diff --git "a/src/test_easy.json" "b/src/test_easy.json"
new file mode 100644--- /dev/null
+++ "b/src/test_easy.json"
@@ -0,0 +1 @@
+[[{"image": "training_data/easy/boards/boards1021/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1021/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1021/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1021/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1021/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 12], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[9, 11], [10, 11], [11, 11], [10, 12], [10, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[6, 6], [6, 7], [7, 7], [7, 8], [8, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 2], [12, 3], [13, 3], [14, 3], [13, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 14], [4, 14], [4, 15], [4, 16], [5, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards727/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards727/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards727/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards727/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards727/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards727/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards727/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards727/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards727/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards727/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards727/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards727/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards727/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards727/step_13.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 16], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[14, 15], [15, 15], [16, 15], [15, 16], [15, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 3], [11, 3], [9, 4], [10, 4], [11, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 10], [5, 11], [6, 11], [7, 11], [6, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 6], [1, 7], [2, 7], [2, 8], [3, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "right", "right", "down", "down", "down", "right", "right", "down", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards1195/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1195/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1195/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1195/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 10], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[10, 9], [11, 9], [12, 9], [11, 10], [11, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 1], [6, 1], [6, 2], [6, 3], [7, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 14], [4, 14], [2, 15], [3, 15], [4, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 4], [13, 5], [14, 5], [15, 5], [14, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "right", "down"]}], [{"image": "training_data/easy/boards/boards1168/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1168/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1168/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1168/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1168/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1168/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1168/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1168/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1168/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1168/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1168/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 15], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[4, 14], [5, 14], [6, 14], [5, 15], [5, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[13, 9], [13, 10], [14, 10], [14, 11], [15, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 3], [7, 4], [8, 4], [9, 4], [8, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 6], [12, 6], [11, 7], [12, 7], [11, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "down", "down", "left", "down", "left", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards3156/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3156/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3156/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3156/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3156/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3156/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3156/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3156/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3156/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3156/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3156/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3156/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3156/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3156/step_13.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 2], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[14, 1], [15, 1], [15, 2], [15, 3], [16, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 10], [10, 10], [11, 10], [10, 11], [10, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 4], [2, 5], [3, 5], [3, 6], [4, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 7], [7, 7], [6, 8], [7, 8], [6, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "right", "up", "up", "right", "right", "right", "up", "right", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards839/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards839/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards839/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards839/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards839/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 8], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[5, 7], [6, 7], [7, 7], [6, 8], [6, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 2], [10, 2], [10, 3], [10, 4], [11, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 13], [12, 14], [13, 14], [13, 15], [14, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[16, 10], [17, 10], [16, 11], [17, 11], [16, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards955/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards955/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards955/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards955/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards955/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards955/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards955/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards955/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards955/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards955/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards955/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards955/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 5], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[15, 4], [16, 4], [17, 4], [16, 5], [16, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 7], [4, 7], [4, 8], [4, 9], [5, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 14], [6, 15], [7, 15], [8, 15], [7, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 10], [11, 10], [10, 11], [11, 11], [10, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "right", "up", "right", "up", "right", "right", "right", "up", "up"]}], [{"image": "training_data/easy/boards/boards903/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards903/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards903/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards903/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards903/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards903/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards903/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards903/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards903/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 7], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[2, 6], [3, 6], [4, 6], [3, 7], [3, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 1], [8, 2], [9, 2], [10, 2], [9, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 15], [16, 15], [15, 16], [16, 16], [15, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 11], [13, 11], [11, 12], [12, 12], [13, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "up", "up", "left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2090/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2090/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2090/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2090/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2090/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2090/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2090/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 11], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[12, 10], [12, 11], [13, 11], [13, 12], [14, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 15], [9, 15], [9, 16], [9, 17], [10, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 1], [15, 2], [16, 2], [17, 2], [16, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 4], [3, 4], [4, 4], [3, 5], [3, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "right", "right", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards598/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards598/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards598/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards598/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards598/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards598/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards598/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards598/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards598/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards598/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards598/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 4], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[14, 3], [15, 3], [14, 4], [15, 4], [14, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 6], [10, 6], [8, 7], [9, 7], [10, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 14], [2, 14], [2, 15], [2, 16], [3, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[4, 10], [5, 10], [6, 10], [5, 11], [5, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "up", "up", "right", "right", "up", "up", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards3483/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3483/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3483/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3483/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3483/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3483/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3483/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3483/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3483/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3483/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3483/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3483/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 13], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[15, 12], [16, 12], [16, 13], [16, 14], [17, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 1], [11, 2], [12, 2], [12, 3], [13, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[4, 7], [6, 7], [4, 8], [5, 8], [6, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 15], [9, 15], [8, 16], [9, 16], [8, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right", "right", "down", "right", "right", "down", "right", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards228/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards228/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards228/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards228/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 10], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[7, 9], [8, 9], [7, 10], [8, 10], [7, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 13], [5, 13], [3, 14], [4, 14], [5, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 1], [14, 2], [15, 2], [15, 3], [16, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[10, 6], [11, 6], [11, 7], [11, 8], [12, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "down"]}], [{"image": "training_data/easy/boards/boards2307/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2307/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2307/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2307/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2307/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2307/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2307/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2307/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2307/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2307/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 11], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[1, 10], [1, 11], [2, 11], [2, 12], [3, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 1], [13, 1], [13, 2], [13, 3], [14, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 7], [4, 8], [5, 8], [6, 8], [5, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 15], [10, 15], [11, 15], [10, 16], [10, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["left", "left", "down", "left", "left", "down", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1711/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1711/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1711/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1711/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1711/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1711/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1711/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1711/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1711/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1711/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1711/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 4], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[3, 3], [5, 3], [3, 4], [4, 4], [5, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[9, 11], [10, 11], [10, 12], [10, 13], [11, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 7], [14, 7], [13, 8], [14, 8], [13, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[15, 14], [15, 15], [16, 15], [16, 16], [17, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "left", "left", "up", "left", "left", "up", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards2223/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2223/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2223/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2223/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2223/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2223/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2223/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 13], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[10, 12], [10, 13], [11, 13], [11, 14], [12, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 7], [15, 7], [13, 8], [14, 8], [15, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 15], [6, 15], [5, 16], [6, 16], [5, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[8, 4], [7, 5], [8, 5], [9, 5], [8, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "down", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards1372/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1372/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1372/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1372/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1372/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1372/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 5], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[7, 4], [9, 4], [7, 5], [8, 5], [9, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 15], [10, 16], [11, 16], [12, 16], [11, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 7], [14, 8], [15, 8], [15, 9], [16, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 11], [2, 11], [3, 11], [2, 12], [2, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards773/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards773/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards773/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 11], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[8, 10], [9, 10], [10, 10], [9, 11], [9, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 13], [17, 13], [15, 14], [16, 14], [17, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[13, 6], [12, 7], [13, 7], [14, 7], [13, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 2], [6, 2], [6, 3], [6, 4], [7, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down"]}], [{"image": "training_data/easy/boards/boards1578/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1578/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 9], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[7, 8], [9, 8], [7, 9], [8, 9], [9, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 14], [3, 14], [3, 15], [3, 16], [4, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[11, 3], [12, 3], [11, 4], [12, 4], [11, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[16, 11], [15, 12], [16, 12], [17, 12], [16, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["left"]}], [{"image": "training_data/easy/boards/boards2976/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2976/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2976/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2976/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2976/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2976/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2976/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2976/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 10], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[15, 9], [14, 10], [15, 10], [16, 10], [15, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[3, 5], [5, 5], [3, 6], [4, 6], [5, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 1], [10, 1], [9, 2], [10, 2], [9, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 14], [12, 14], [12, 15], [12, 16], [13, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["down", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1910/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1910/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1910/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1910/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1910/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1910/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1910/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1910/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1910/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1910/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1910/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1910/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 13], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[1, 12], [1, 13], [2, 13], [2, 14], [3, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[8, 7], [9, 7], [8, 8], [9, 8], [8, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 4], [11, 5], [12, 5], [13, 5], [12, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 1], [16, 1], [14, 2], [15, 2], [16, 2]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "down", "left", "left", "down", "left", "down", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards1870/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1870/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1870/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1870/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1870/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1870/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 7], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[11, 6], [11, 7], [12, 7], [12, 8], [13, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 9], [16, 9], [17, 9], [16, 10], [16, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 3], [9, 3], [9, 4], [9, 5], [10, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[4, 15], [5, 15], [4, 16], [5, 16], [4, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "right", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards1197/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1197/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1197/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1197/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1197/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1197/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1197/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1197/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1197/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1197/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1197/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 16], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[5, 15], [6, 15], [7, 15], [6, 16], [6, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 6], [1, 7], [2, 7], [3, 7], [2, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 11], [13, 11], [12, 12], [13, 12], [12, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "down", "left", "down", "down", "down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards2082/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2082/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2082/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2082/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2082/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2082/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2082/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 14], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[9, 13], [9, 14], [10, 14], [10, 15], [11, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 10], [13, 10], [13, 11], [13, 12], [14, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[16, 6], [17, 6], [16, 7], [17, 7], [16, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 2], [7, 2], [8, 2], [7, 3], [7, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards1894/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1894/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1894/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1894/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1894/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1894/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 9], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[3, 8], [3, 9], [4, 9], [4, 10], [5, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 12], [12, 12], [11, 13], [12, 13], [11, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[6, 15], [8, 15], [6, 16], [7, 16], [8, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 4], [14, 4], [15, 4], [14, 5], [14, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1820/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1820/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1820/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1820/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1820/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1820/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1820/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1820/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1820/step_8.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 15], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[6, 14], [6, 15], [7, 15], [7, 16], [8, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 10], [5, 10], [3, 11], [4, 11], [5, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 1], [9, 2], [10, 2], [11, 2], [10, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 5], [14, 5], [13, 6], [14, 6], [13, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "left", "left", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards712/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards712/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards712/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards712/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards712/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards712/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[7, 12], [8, 12], [9, 12], [8, 13], [8, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 1], [12, 1], [12, 2], [12, 3], [13, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 15], [3, 15], [1, 16], [2, 16], [3, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 6], [4, 7], [5, 7], [6, 7], [5, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards1243/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1243/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1243/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1243/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1243/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1243/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1243/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1243/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1243/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1243/step_9.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1243/step_10.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 5], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[14, 4], [16, 4], [14, 5], [15, 5], [16, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 13], [3, 13], [2, 14], [3, 14], [2, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[9, 7], [10, 7], [10, 8], [10, 9], [11, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 10], [5, 10], [6, 10], [5, 11], [5, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "up", "up", "right", "right", "up", "up", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards23/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards23/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards23/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards23/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards23/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards23/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards23/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards23/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards23/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards23/step_9.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards23/step_10.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 5], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[15, 4], [16, 4], [15, 5], [16, 5], [15, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 15], [8, 16], [9, 16], [10, 16], [9, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[5, 12], [6, 12], [7, 12], [6, 13], [6, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 1], [12, 1], [12, 2], [12, 3], [13, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "up", "up", "right", "right", "right", "up", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards3389/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3389/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3389/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3389/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3389/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3389/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3389/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 13], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[6, 12], [7, 12], [7, 13], [7, 14], [8, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 2], [9, 3], [10, 3], [11, 3], [10, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 5], [1, 6], [2, 6], [2, 7], [3, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 15], [15, 15], [13, 16], [14, 16], [15, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["down", "down", "left", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards3283/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3283/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3283/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3283/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3283/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3283/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3283/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3283/step_7.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3283/step_8.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 6], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[3, 5], [4, 5], [4, 6], [4, 7], [5, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 10], [8, 10], [7, 11], [8, 11], [7, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 1], [10, 2], [11, 2], [12, 2], [11, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 13], [15, 13], [13, 14], [14, 14], [15, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "up", "left", "left", "left", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards92/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards92/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards92/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards92/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards92/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards92/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards92/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards92/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards92/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards92/step_9.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards92/step_10.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 5], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[15, 4], [16, 4], [15, 5], [16, 5], [15, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 11], [2, 11], [3, 11], [2, 12], [2, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 7], [7, 8], [8, 8], [9, 8], [8, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 14], [4, 15], [5, 15], [5, 16], [6, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "up", "up", "right", "right", "right", "right", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards508/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards508/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards508/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards508/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards508/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards508/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards508/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards508/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards508/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 13], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[5, 12], [6, 12], [5, 13], [6, 13], [5, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[10, 2], [11, 2], [12, 2], [11, 3], [11, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 15], [15, 15], [13, 16], [14, 16], [15, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 9], [7, 10], [8, 10], [9, 10], [8, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "down", "left", "down", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards3258/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3258/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3258/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3258/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3258/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3258/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3258/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 10], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[13, 9], [14, 9], [14, 10], [14, 11], [15, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[4, 5], [6, 5], [4, 6], [5, 6], [6, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 13], [11, 13], [10, 14], [11, 14], [10, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 2], [1, 3], [2, 3], [2, 4], [3, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "right", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards1570/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1570/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1570/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1570/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1570/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1570/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1570/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 11], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[12, 10], [14, 10], [12, 11], [13, 11], [14, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 15], [9, 15], [9, 16], [9, 17], [10, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 1], [15, 2], [16, 2], [17, 2], [16, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 4], [3, 4], [4, 4], [3, 5], [3, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "down", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards879/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards879/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards879/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards879/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards879/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards879/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards879/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards879/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards879/step_8.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards879/step_9.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 3], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[5, 2], [6, 2], [7, 2], [6, 3], [6, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 9], [15, 9], [13, 10], [14, 10], [15, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[11, 15], [10, 16], [11, 16], [12, 16], [11, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 6], [3, 6], [3, 7], [3, 8], [4, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left", "up", "up", "up", "up", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards3462/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3462/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3462/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3462/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 12], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[8, 11], [9, 11], [9, 12], [9, 13], [10, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 3], [2, 4], [3, 4], [4, 4], [3, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 15], [16, 15], [14, 16], [15, 16], [16, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[12, 8], [13, 8], [12, 9], [13, 9], [12, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1889/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1889/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1889/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1889/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1889/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1889/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 11], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[11, 10], [11, 11], [12, 11], [12, 12], [13, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 1], [2, 2], [3, 2], [4, 2], [3, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 7], [15, 7], [15, 8], [15, 9], [16, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 4], [8, 4], [9, 4], [8, 5], [8, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "right", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards1873/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1873/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1873/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1873/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1873/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 12], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[7, 11], [7, 12], [8, 12], [8, 13], [9, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 7], [2, 7], [2, 8], [2, 9], [3, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[5, 1], [6, 1], [5, 2], [6, 2], [5, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 4], [13, 4], [14, 4], [13, 5], [13, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards2209/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2209/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2209/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2209/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2209/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2209/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2209/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2209/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 16], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[8, 15], [8, 16], [9, 16], [9, 17], [10, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 8], [14, 8], [13, 9], [14, 9], [13, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 3], [3, 3], [1, 4], [2, 4], [3, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[16, 12], [15, 13], [16, 13], [17, 13], [16, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["down", "down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards677/step_0.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards677/step_1.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards677/step_2.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards677/step_3.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards677/step_4.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards677/step_5.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards677/step_6.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards677/step_7.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards677/step_8.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards677/step_9.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards677/step_10.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 16], "target_shape": "T", "target_color": "red", "info": [{"piece_grids": [[5, 15], [6, 15], [7, 15], [6, 16], [6, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 6], [1, 7], [2, 7], [3, 7], [2, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 11], [13, 11], [12, 12], [13, 12], [12, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "down", "down", "left", "left", "down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards1660/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1660/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1660/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1660/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1660/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 6], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[7, 5], [9, 5], [7, 6], [8, 6], [9, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 10], [11, 11], [12, 11], [13, 11], [12, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 15], [4, 15], [4, 16], [4, 17], [5, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 1], [15, 1], [16, 1], [15, 2], [15, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["up", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards1231/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1231/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1231/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1231/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1231/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1231/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1231/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1231/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 3], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[7, 2], [9, 2], [7, 3], [8, 3], [9, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 12], [14, 13], [15, 13], [15, 14], [16, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 9], [12, 9], [12, 10], [12, 11], [13, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 6], [4, 6], [3, 7], [4, 7], [3, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["left", "up", "up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards1999/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1999/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1999/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1999/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1999/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1999/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 4], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[8, 3], [8, 4], [9, 4], [9, 5], [10, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 7], [12, 7], [13, 7], [12, 8], [12, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[4, 15], [5, 15], [5, 16], [5, 17], [6, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2937/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2937/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2937/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 7], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[9, 6], [8, 7], [9, 7], [10, 7], [9, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 1], [4, 1], [2, 2], [3, 2], [4, 2]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[5, 10], [5, 11], [6, 11], [6, 12], [7, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 14], [15, 14], [15, 15], [15, 16], [16, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "up"]}], [{"image": "training_data/easy/boards/boards61/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards61/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards61/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards61/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards61/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards61/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 7], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[6, 6], [7, 6], [6, 7], [7, 7], [6, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 2], [13, 2], [14, 2], [13, 3], [13, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 11], [11, 11], [9, 12], [10, 12], [11, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 15], [2, 16], [3, 16], [4, 16], [3, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["left", "up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards2273/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2273/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2273/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2273/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2273/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2273/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2273/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 5], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[6, 4], [6, 5], [7, 5], [7, 6], [8, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 9], [12, 9], [12, 10], [12, 11], [13, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 1], [3, 2], [4, 2], [5, 2], [4, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 12], [15, 12], [16, 12], [15, 13], [15, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "left", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards3410/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3410/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3410/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3410/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3410/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3410/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3410/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3410/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3410/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3410/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3410/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 14], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[13, 13], [14, 13], [14, 14], [14, 15], [15, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 6], [9, 6], [7, 7], [8, 7], [9, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 3], [11, 3], [12, 3], [11, 4], [11, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 10], [6, 10], [5, 11], [6, 11], [5, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "right", "down", "right", "right", "right", "right", "down"]}], [{"image": "training_data/easy/boards/boards2217/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2217/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2217/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2217/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2217/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2217/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2217/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2217/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 11], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[3, 10], [3, 11], [4, 11], [4, 12], [5, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 2], [12, 2], [11, 3], [12, 3], [11, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 13], [17, 13], [15, 14], [16, 14], [17, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 7], [8, 7], [9, 7], [8, 8], [8, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "down", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2420/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2420/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2420/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2420/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2420/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2420/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 10], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[5, 9], [4, 10], [5, 10], [6, 10], [5, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 3], [11, 3], [10, 4], [11, 4], [10, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 13], [13, 13], [14, 13], [13, 14], [13, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[15, 6], [17, 6], [15, 7], [16, 7], [17, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1857/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1857/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1857/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1857/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1857/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1857/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1857/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1857/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1857/step_8.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1857/step_9.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 3], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[5, 2], [5, 3], [6, 3], [6, 4], [7, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 14], [11, 15], [12, 15], [13, 15], [12, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 9], [3, 9], [3, 10], [3, 11], [4, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 5], [17, 5], [15, 6], [16, 6], [17, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "left", "up", "up", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards349/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards349/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards349/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards349/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards349/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards349/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards349/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards349/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards349/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards349/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards349/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards349/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards349/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards349/step_13.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 3], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[2, 2], [3, 2], [2, 3], [3, 3], [2, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 8], [12, 9], [13, 9], [13, 10], [14, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 11], [9, 11], [7, 12], [8, 12], [9, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 5], [15, 6], [16, 6], [17, 6], [16, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "left", "left", "up", "left", "up", "left", "left", "up", "up", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards2743/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2743/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2743/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2743/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2743/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2743/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2743/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 13], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[11, 12], [10, 13], [11, 13], [12, 13], [11, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 7], [13, 8], [14, 8], [14, 9], [15, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 15], [6, 15], [5, 16], [6, 16], [5, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[7, 4], [8, 4], [8, 5], [8, 6], [9, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "down", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards1190/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1190/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1190/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1190/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1190/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1190/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1190/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1190/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1190/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1190/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1190/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 14], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[3, 13], [4, 13], [5, 13], [4, 14], [4, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[11, 1], [12, 1], [12, 2], [12, 3], [13, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[7, 6], [6, 7], [7, 7], [8, 7], [7, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 10], [15, 11], [16, 11], [16, 12], [17, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "left", "left", "down", "down", "down", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards1476/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1476/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1476/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1476/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1476/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1476/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1476/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1476/step_7.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1476/step_8.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1476/step_9.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1476/step_10.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1476/step_11.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1476/step_12.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1476/step_13.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1476/step_14.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 16], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[15, 15], [17, 15], [15, 16], [16, 16], [17, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 4], [11, 5], [12, 5], [12, 6], [13, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[8, 8], [9, 8], [9, 9], [9, 10], [10, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 1], [3, 2], [4, 2], [5, 2], [4, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "down", "right", "down", "down", "right", "right", "down", "down", "down", "right", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards2887/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2887/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2887/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2887/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2887/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2887/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2887/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2887/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 5], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[6, 4], [5, 5], [6, 5], [7, 5], [6, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 13], [11, 13], [10, 14], [11, 14], [10, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["up", "up", "up", "left", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards3098/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3098/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3098/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3098/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3098/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3098/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3098/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3098/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 4], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[10, 3], [11, 3], [11, 4], [11, 5], [12, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 8], [17, 8], [15, 9], [16, 9], [17, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 11], [7, 11], [8, 11], [7, 12], [7, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 14], [5, 14], [4, 15], [5, 15], [4, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "right", "up", "up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards620/step_0.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards620/step_1.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards620/step_2.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards620/step_3.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards620/step_4.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 6], "target_shape": "T", "target_color": "red", "info": [{"piece_grids": [[7, 5], [8, 5], [9, 5], [8, 6], [8, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 10], [12, 10], [12, 11], [12, 12], [13, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 15], [5, 15], [3, 16], [4, 16], [5, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 1], [14, 2], [15, 2], [16, 2], [15, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards3033/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3033/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3033/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3033/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3033/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3033/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3033/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3033/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3033/step_8.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 4], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[5, 3], [6, 3], [6, 4], [6, 5], [7, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 14], [15, 14], [13, 15], [14, 15], [15, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[1, 8], [2, 8], [3, 8], [2, 9], [2, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 11], [10, 11], [9, 12], [10, 12], [9, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "left", "left", "up", "left", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2872/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2872/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2872/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2872/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2872/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2872/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2872/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2872/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2872/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2872/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2872/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 16], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[6, 15], [5, 16], [6, 16], [7, 16], [6, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[8, 4], [8, 5], [9, 5], [9, 6], [10, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 7], [12, 7], [12, 8], [12, 9], [13, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 1], [17, 1], [16, 2], [17, 2], [16, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "left", "down", "down", "down", "left", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards394/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards394/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards394/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards394/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards394/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards394/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards394/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards394/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards394/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards394/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 15], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[6, 14], [7, 14], [6, 15], [7, 15], [6, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 2], [15, 3], [16, 3], [17, 3], [16, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 5], [11, 6], [12, 6], [12, 7], [13, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 9], [3, 9], [3, 10], [3, 11], [4, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "left", "down", "down", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards668/step_0.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards668/step_1.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards668/step_2.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards668/step_3.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards668/step_4.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards668/step_5.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards668/step_6.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards668/step_7.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards668/step_8.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards668/step_9.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 14], "target_shape": "T", "target_color": "red", "info": [{"piece_grids": [[4, 13], [5, 13], [6, 13], [5, 14], [5, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[12, 4], [13, 4], [13, 5], [13, 6], [14, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 7], [1, 8], [2, 8], [3, 8], [2, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 10], [10, 10], [8, 11], [9, 11], [10, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "down", "left", "left", "left", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1742/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1742/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1742/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1742/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1742/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 6], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[9, 5], [11, 5], [9, 6], [10, 6], [11, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 14], [6, 14], [7, 14], [6, 15], [6, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 9], [15, 9], [14, 10], [15, 10], [14, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [1, 2], [2, 2], [3, 2], [2, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards3316/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3316/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3316/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3316/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3316/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3316/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3316/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3316/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3316/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3316/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3316/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 12], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[1, 11], [2, 11], [2, 12], [2, 13], [3, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 15], [13, 15], [11, 16], [12, 16], [13, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 8], [5, 8], [6, 8], [5, 9], [5, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 4], [9, 4], [8, 5], [9, 5], [8, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "down", "left", "left", "left", "left", "down", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards1627/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1627/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1627/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1627/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1627/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1627/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1627/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1627/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 14], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[6, 13], [8, 13], [6, 14], [7, 14], [8, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 9], [3, 9], [4, 9], [3, 10], [3, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 1], [15, 1], [14, 2], [15, 2], [14, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[10, 4], [11, 4], [11, 5], [11, 6], [12, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "down", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards1263/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1263/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1263/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1263/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1263/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1263/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1263/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1263/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1263/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1263/step_9.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1263/step_10.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 2], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[11, 1], [13, 1], [11, 2], [12, 2], [13, 2]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 10], [5, 11], [6, 11], [7, 11], [6, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 15], [3, 15], [2, 16], [3, 16], [2, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 6], [16, 6], [17, 6], [16, 7], [16, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "right", "up", "up", "up", "right", "up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards1042/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1042/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1042/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1042/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1042/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1042/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1042/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 14], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[9, 13], [10, 13], [11, 13], [10, 14], [10, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 10], [14, 10], [12, 11], [13, 11], [14, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[16, 6], [17, 6], [16, 7], [17, 7], [16, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 2], [6, 3], [7, 3], [8, 3], [7, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards245/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards245/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards245/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards245/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards245/step_4.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 9], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[13, 8], [14, 8], [13, 9], [14, 9], [13, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 3], [9, 3], [10, 3], [9, 4], [9, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[4, 15], [6, 15], [4, 16], [5, 16], [6, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 12], [16, 12], [16, 13], [16, 14], [17, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1665/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1665/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1665/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1665/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1665/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1665/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1665/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1665/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 14], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[6, 13], [8, 13], [6, 14], [7, 14], [8, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 1], [10, 2], [11, 2], [11, 3], [12, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[3, 4], [2, 5], [3, 5], [4, 5], [3, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 7], [14, 7], [15, 7], [14, 8], [14, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "down", "down", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards2083/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2083/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2083/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2083/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2083/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2083/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2083/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2083/step_7.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2083/step_8.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2083/step_9.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2083/step_10.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2083/step_11.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2083/step_12.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 14], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[1, 13], [1, 14], [2, 14], [2, 15], [3, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[6, 5], [5, 6], [6, 6], [7, 6], [6, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 10], [10, 10], [9, 11], [10, 11], [9, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 2], [15, 2], [13, 3], [14, 3], [15, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "down", "left", "down", "down", "left", "left", "down", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards1821/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1821/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1821/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1821/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1821/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1821/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 8], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[12, 7], [12, 8], [13, 8], [13, 9], [14, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 1], [7, 1], [5, 2], [6, 2], [7, 2]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 4], [16, 4], [16, 5], [16, 6], [17, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 10], [10, 10], [11, 10], [10, 11], [10, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards2255/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2255/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2255/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2255/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2255/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2255/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2255/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2255/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2255/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2255/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2255/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2255/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 5], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[15, 4], [15, 5], [16, 5], [16, 6], [17, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[4, 7], [3, 8], [4, 8], [5, 8], [4, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[6, 14], [7, 14], [8, 14], [7, 15], [7, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 10], [11, 10], [10, 11], [11, 11], [10, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "right", "up", "up", "right", "up", "right", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards2088/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2088/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2088/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2088/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2088/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2088/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2088/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2088/step_7.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2088/step_8.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2088/step_9.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2088/step_10.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2088/step_11.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2088/step_12.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2088/step_13.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 2], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[2, 1], [2, 2], [3, 2], [3, 3], [4, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 14], [8, 14], [7, 15], [8, 15], [7, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 7], [12, 7], [12, 8], [12, 9], [13, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 10], [16, 10], [14, 11], [15, 11], [16, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "left", "up", "left", "up", "left", "up", "left", "up", "up", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards1227/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1227/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1227/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1227/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1227/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1227/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1227/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1227/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 12], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[4, 11], [6, 11], [4, 12], [5, 12], [6, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 14], [14, 15], [15, 15], [15, 16], [16, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[2, 6], [1, 7], [2, 7], [3, 7], [2, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 3], [11, 3], [11, 4], [11, 5], [12, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "down", "left", "left", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards2521/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2521/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2521/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2521/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2521/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2521/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2521/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 15], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[9, 14], [8, 15], [9, 15], [10, 15], [9, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 4], [5, 4], [6, 4], [5, 5], [5, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 7], [15, 7], [14, 8], [15, 8], [14, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards3012/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3012/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3012/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 9], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[6, 8], [7, 8], [7, 9], [7, 10], [8, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 5], [16, 5], [14, 6], [15, 6], [16, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[10, 11], [11, 11], [12, 11], [11, 12], [11, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 15], [4, 15], [3, 16], [4, 16], [3, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["left", "left"]}], [{"image": "training_data/easy/boards/boards355/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards355/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards355/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 9], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[7, 8], [8, 8], [7, 9], [8, 9], [7, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 4], [14, 5], [15, 5], [15, 6], [16, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 14], [13, 14], [11, 15], [12, 15], [13, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 11], [3, 11], [3, 12], [3, 13], [4, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["left", "left"]}], [{"image": "training_data/easy/boards/boards3466/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3466/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3466/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3466/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3466/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3466/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3466/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3466/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3466/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3466/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3466/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3466/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3466/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3466/step_13.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 16], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[14, 15], [15, 15], [15, 16], [15, 17], [16, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 12], [8, 12], [7, 13], [8, 13], [7, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 3], [2, 3], [3, 3], [2, 4], [2, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[9, 9], [9, 10], [10, 10], [10, 11], [11, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "right", "down", "right", "down", "down", "down", "down", "down", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards1741/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1741/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1741/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1741/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1741/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1741/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1741/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 15], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[8, 14], [10, 14], [8, 15], [9, 15], [10, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 4], [5, 4], [6, 4], [5, 5], [5, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 7], [15, 7], [14, 8], [15, 8], [14, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 10], [2, 10], [2, 11], [2, 12], [3, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards2281/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2281/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2281/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 7], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[8, 6], [8, 7], [9, 7], [9, 8], [10, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 11], [14, 12], [15, 12], [16, 12], [15, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 14], [7, 14], [6, 15], [7, 15], [6, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 2], [3, 2], [4, 2], [3, 3], [3, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}], "path": ["up", "up"]}], [{"image": "training_data/easy/boards/boards655/step_0.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards655/step_1.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards655/step_2.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards655/step_3.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards655/step_4.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards655/step_5.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards655/step_6.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards655/step_7.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 3], "target_shape": "T", "target_color": "red", "info": [{"piece_grids": [[9, 2], [10, 2], [11, 2], [10, 3], [10, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 10], [2, 10], [2, 11], [2, 12], [3, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 6], [15, 6], [14, 7], [15, 7], [14, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 13], [4, 14], [5, 14], [6, 14], [5, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "right", "up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2999/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2999/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2999/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2999/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2999/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2999/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2999/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2999/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 6], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[13, 5], [12, 6], [13, 6], [14, 6], [13, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 12], [9, 12], [9, 13], [9, 14], [10, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 8], [17, 8], [16, 9], [17, 9], [16, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 1], [5, 2], [6, 2], [6, 3], [7, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "right", "up", "up", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards397/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards397/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards397/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards397/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards397/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards397/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards397/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards397/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 11], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[4, 10], [5, 10], [4, 11], [5, 11], [4, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 2], [11, 2], [12, 2], [11, 3], [11, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 13], [15, 14], [16, 14], [16, 15], [17, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 7], [7, 8], [8, 8], [9, 8], [8, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "left", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards880/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards880/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards880/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards880/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards880/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 6], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[7, 5], [8, 5], [9, 5], [8, 6], [8, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 10], [12, 10], [12, 11], [12, 12], [13, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 15], [5, 15], [3, 16], [4, 16], [5, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 1], [14, 2], [15, 2], [16, 2], [15, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards111/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards111/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards111/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards111/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards111/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards111/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "P", "target_color": "blue", "info": [{"piece_grids": [[8, 12], [9, 12], [8, 13], [9, 13], [8, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 9], [3, 9], [4, 9], [3, 10], [3, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 4], [13, 5], [14, 5], [15, 5], [14, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[10, 1], [11, 1], [11, 2], [11, 3], [12, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards3122/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3122/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3122/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3122/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3122/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3122/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3122/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 14], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[9, 13], [10, 13], [10, 14], [10, 15], [11, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 10], [14, 10], [12, 11], [13, 11], [14, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[16, 6], [17, 6], [16, 7], [17, 7], [16, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 2], [7, 2], [8, 2], [7, 3], [7, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards2262/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2262/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2262/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2262/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2262/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 6], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[9, 5], [9, 6], [10, 6], [10, 7], [11, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 14], [6, 14], [7, 14], [6, 15], [6, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 9], [15, 9], [14, 10], [15, 10], [14, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [1, 2], [2, 2], [3, 2], [2, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards2548/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2548/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2548/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2548/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2548/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2548/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2548/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2548/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2548/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2548/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 12], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[3, 11], [2, 12], [3, 12], [4, 12], [3, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 3], [8, 3], [9, 3], [8, 4], [8, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 15], [13, 15], [11, 16], [12, 16], [13, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 7], [17, 7], [16, 8], [17, 8], [16, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "left", "left", "left", "left", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards558/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards558/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards558/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards558/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards558/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards558/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards558/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards558/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 4], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[11, 3], [12, 3], [11, 4], [12, 4], [11, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 15], [5, 16], [6, 16], [7, 16], [6, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 11], [3, 11], [3, 12], [3, 13], [4, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 7], [17, 7], [15, 8], [16, 8], [17, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards1528/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1528/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1528/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1528/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 10], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[6, 9], [8, 9], [6, 10], [7, 10], [8, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 13], [4, 13], [4, 14], [4, 15], [5, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 1], [14, 2], [15, 2], [15, 3], [16, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 6], [10, 7], [11, 7], [12, 7], [11, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "left"]}], [{"image": "training_data/easy/boards/boards84/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards84/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards84/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards84/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards84/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards84/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards84/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 11], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[5, 10], [6, 10], [5, 11], [6, 11], [5, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 15], [15, 15], [13, 16], [14, 16], [15, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 2], [9, 2], [9, 3], [9, 4], [10, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 6], [1, 7], [2, 7], [3, 7], [2, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "left", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards1949/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1949/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1949/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1949/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1949/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1949/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1949/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1949/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 16], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[8, 15], [8, 16], [9, 16], [9, 17], [10, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 8], [14, 8], [13, 9], [14, 9], [13, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 3], [3, 3], [1, 4], [2, 4], [3, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[16, 12], [15, 13], [16, 13], [17, 13], [16, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["down", "down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards2248/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2248/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2248/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2248/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2248/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2248/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2248/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2248/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2248/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2248/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 3], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[11, 2], [11, 3], [12, 3], [12, 4], [13, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 11], [7, 11], [6, 12], [7, 12], [6, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 8], [17, 8], [15, 9], [16, 9], [17, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 14], [8, 15], [9, 15], [10, 15], [9, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["up", "up", "up", "up", "right", "right", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards1858/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1858/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1858/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1858/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1858/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1858/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1858/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1858/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 4], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[10, 3], [10, 4], [11, 4], [11, 5], [12, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 15], [6, 15], [7, 15], [6, 16], [6, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 11], [2, 12], [3, 12], [4, 12], [3, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 7], [16, 7], [16, 8], [16, 9], [17, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "right", "up", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2415/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2415/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2415/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2415/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2415/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2415/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 7], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[6, 6], [5, 7], [6, 7], [7, 7], [6, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 2], [1, 3], [2, 3], [2, 4], [3, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 10], [17, 10], [15, 11], [16, 11], [17, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[12, 15], [13, 15], [13, 16], [13, 17], [14, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards1842/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1842/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1842/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1842/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1842/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1842/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1842/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1842/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 13], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[5, 12], [5, 13], [6, 13], [6, 14], [7, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 3], [16, 3], [17, 3], [16, 4], [16, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 6], [2, 6], [2, 7], [2, 8], [3, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 9], [13, 9], [12, 10], [13, 10], [12, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "left", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards917/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards917/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards917/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards917/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards917/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards917/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards917/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards917/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 11], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[3, 10], [4, 10], [5, 10], [4, 11], [4, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 2], [12, 2], [11, 3], [12, 3], [11, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 13], [15, 14], [16, 14], [16, 15], [17, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 7], [7, 8], [8, 8], [9, 8], [8, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "down", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards596/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards596/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards596/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards596/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards596/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards596/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards596/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards596/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 10], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[3, 9], [4, 9], [3, 10], [4, 10], [3, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 3], [15, 4], [16, 4], [16, 5], [17, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[6, 14], [8, 14], [6, 15], [7, 15], [8, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 6], [10, 7], [11, 7], [12, 7], [11, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "left", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards2268/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2268/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2268/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2268/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2268/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2268/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2268/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2268/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2268/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2268/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2268/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2268/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 13], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[15, 12], [15, 13], [16, 13], [16, 14], [17, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 6], [7, 6], [6, 7], [7, 7], [6, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 15], [12, 15], [10, 16], [11, 16], [12, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 2], [1, 3], [2, 3], [3, 3], [2, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "right", "down", "right", "down", "right", "down", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards1203/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1203/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1203/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1203/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1203/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1203/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1203/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1203/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1203/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 6], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[3, 5], [5, 5], [3, 6], [4, 6], [5, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 10], [8, 10], [7, 11], [8, 11], [7, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 1], [10, 2], [11, 2], [12, 2], [11, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 13], [14, 13], [14, 14], [14, 15], [15, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "up", "left", "up", "left", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards1510/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1510/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1510/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1510/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 11], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[9, 10], [11, 10], [9, 11], [10, 11], [11, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 1], [13, 2], [14, 2], [15, 2], [14, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 13], [1, 14], [2, 14], [2, 15], [3, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 5], [6, 5], [6, 6], [6, 7], [7, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "right"]}], [{"image": "training_data/easy/boards/boards1518/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1518/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1518/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1518/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1518/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1518/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1518/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1518/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1518/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1518/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1518/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1518/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1518/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1518/step_13.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 15], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[1, 14], [3, 14], [1, 15], [2, 15], [3, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 9], [5, 10], [6, 10], [6, 11], [7, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 6], [14, 6], [15, 6], [14, 7], [14, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 3], [10, 3], [10, 4], [10, 5], [11, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "left", "left", "down", "left", "left", "down", "down", "left", "down", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards2085/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2085/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2085/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2085/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2085/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2085/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 4], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[8, 3], [8, 4], [9, 4], [9, 5], [10, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 14], [4, 15], [5, 15], [6, 15], [5, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[2, 6], [3, 6], [2, 7], [3, 7], [2, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 11], [14, 11], [15, 11], [14, 12], [14, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards142/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards142/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards142/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards142/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards142/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards142/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards142/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards142/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards142/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 10], "target_shape": "P", "target_color": "blue", "info": [{"piece_grids": [[16, 9], [17, 9], [16, 10], [17, 10], [16, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[12, 1], [12, 2], [13, 2], [13, 3], [14, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 5], [4, 6], [5, 6], [6, 6], [5, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[1, 13], [2, 13], [2, 14], [2, 15], [3, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "right", "right", "down", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1945/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1945/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1945/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1945/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1945/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1945/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1945/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1945/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 15], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[9, 14], [9, 15], [10, 15], [10, 16], [11, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 1], [4, 1], [3, 2], [4, 2], [3, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 7], [16, 7], [17, 7], [16, 8], [16, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 11], [7, 11], [5, 12], [6, 12], [7, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "down", "down", "right", "down", "down"]}], [{"image": "training_data/easy/boards/boards259/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards259/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards259/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards259/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards259/step_4.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards259/step_5.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards259/step_6.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards259/step_7.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards259/step_8.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 2], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[10, 1], [11, 1], [10, 2], [11, 2], [10, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[4, 7], [5, 7], [6, 7], [5, 8], [5, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 4], [16, 4], [14, 5], [15, 5], [16, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 12], [1, 13], [2, 13], [2, 14], [3, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up", "up", "up", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2278/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2278/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2278/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2278/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2278/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2278/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2278/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2278/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2278/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 10], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[1, 9], [1, 10], [2, 10], [2, 11], [3, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 5], [9, 5], [7, 6], [8, 6], [9, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 1], [12, 1], [11, 2], [12, 2], [11, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 14], [14, 14], [14, 15], [14, 16], [15, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "left", "left", "down", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards784/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards784/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards784/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards784/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards784/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards784/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards784/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards784/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards784/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards784/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards784/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards784/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards784/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards784/step_13.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 2], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[14, 1], [15, 1], [16, 1], [15, 2], [15, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 12], [2, 13], [3, 13], [3, 14], [4, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[6, 4], [7, 4], [6, 5], [7, 5], [6, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[9, 15], [8, 16], [9, 16], [10, 16], [9, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right", "up", "right", "right", "right", "up", "up", "up", "up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards1572/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1572/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1572/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1572/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1572/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1572/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1572/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1572/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1572/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1572/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1572/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 16], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[5, 15], [7, 15], [5, 16], [6, 16], [7, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[8, 4], [8, 5], [9, 5], [9, 6], [10, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 7], [11, 8], [12, 8], [13, 8], [12, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 1], [17, 1], [16, 2], [17, 2], [16, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "down", "down", "down", "down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards1946/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1946/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1946/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1946/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1946/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1946/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1946/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 12], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[11, 11], [11, 12], [12, 12], [12, 13], [13, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 4], [10, 4], [9, 5], [10, 5], [9, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 14], [6, 14], [7, 14], [6, 15], [6, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 8], [1, 9], [2, 9], [3, 9], [2, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "down", "down", "right", "right", "down"]}], [{"image": "training_data/easy/boards/boards3051/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3051/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3051/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3051/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3051/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3051/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3051/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3051/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 3], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[7, 2], [8, 2], [8, 3], [8, 4], [9, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 12], [14, 13], [15, 13], [15, 14], [16, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 9], [13, 9], [11, 10], [12, 10], [13, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 6], [4, 6], [3, 7], [4, 7], [3, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "up", "up", "up", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards1334/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1334/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1334/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1334/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1334/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1334/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1334/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1334/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 3], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[7, 2], [9, 2], [7, 3], [8, 3], [9, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 12], [13, 12], [13, 13], [13, 14], [14, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[2, 15], [3, 15], [2, 16], [3, 16], [2, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[16, 5], [15, 6], [16, 6], [17, 6], [16, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards923/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards923/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards923/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards923/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards923/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards923/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards923/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 13], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[10, 12], [11, 12], [12, 12], [11, 13], [11, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 7], [13, 8], [14, 8], [14, 9], [15, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 15], [6, 15], [5, 16], [6, 16], [5, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[7, 4], [8, 4], [8, 5], [8, 6], [9, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1185/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1185/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1185/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1185/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1185/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1185/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1185/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 11], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[12, 10], [13, 10], [14, 10], [13, 11], [13, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 14], [7, 14], [6, 15], [7, 15], [6, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[8, 7], [8, 8], [9, 8], [9, 9], [10, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 1], [2, 2], [3, 2], [4, 2], [3, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "right", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards2471/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2471/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2471/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2471/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2471/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2471/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2471/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 11], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[13, 10], [12, 11], [13, 11], [14, 11], [13, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 6], [6, 6], [5, 7], [6, 7], [5, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 3], [17, 3], [15, 4], [16, 4], [17, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 14], [10, 14], [10, 15], [10, 16], [11, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["down", "right", "down", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1357/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1357/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1357/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1357/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1357/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1357/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1357/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1357/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1357/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 14], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[5, 13], [7, 13], [5, 14], [6, 14], [7, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[8, 10], [9, 10], [10, 10], [9, 11], [9, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 3], [16, 3], [16, 4], [16, 5], [17, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[12, 7], [12, 8], [13, 8], [13, 9], [14, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "down", "down", "down", "down", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards2665/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2665/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2665/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2665/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2665/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2665/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2665/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2665/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2665/step_8.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2665/step_9.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2665/step_10.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2665/step_11.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2665/step_12.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 14], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[16, 13], [15, 14], [16, 14], [17, 14], [16, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 10], [6, 10], [5, 11], [6, 11], [5, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 1], [10, 1], [11, 1], [10, 2], [10, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 6], [13, 6], [13, 7], [13, 8], [14, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "down", "down", "right", "down", "down", "right", "down", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1434/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1434/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1434/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1434/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1434/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1434/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1434/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1434/step_7.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1434/step_8.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1434/step_9.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 15], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[5, 14], [7, 14], [5, 15], [6, 15], [7, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 2], [16, 2], [17, 2], [16, 3], [16, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 5], [11, 6], [12, 6], [12, 7], [13, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 9], [2, 10], [3, 10], [4, 10], [3, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "down", "down", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards1813/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1813/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1813/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 11], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[8, 10], [8, 11], [9, 11], [9, 12], [10, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 13], [16, 13], [16, 14], [16, 15], [17, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[12, 6], [13, 6], [14, 6], [13, 7], [13, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 2], [5, 3], [6, 3], [7, 3], [6, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down"]}], [{"image": "training_data/easy/boards/boards2764/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2764/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2764/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2764/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 8], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[11, 7], [10, 8], [11, 8], [12, 8], [11, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 1], [15, 1], [13, 2], [14, 2], [15, 2]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 15], [6, 15], [7, 15], [6, 16], [6, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 10], [1, 11], [2, 11], [2, 12], [3, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "up", "right"]}], [{"image": "training_data/easy/boards/boards176/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards176/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards176/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards176/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards176/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards176/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards176/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards176/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards176/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards176/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards176/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards176/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards176/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards176/step_13.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards176/step_14.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 16], "target_shape": "P", "target_color": "blue", "info": [{"piece_grids": [[16, 15], [17, 15], [16, 16], [17, 16], [16, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 4], [11, 5], [12, 5], [12, 6], [13, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[8, 8], [10, 8], [8, 9], [9, 9], [10, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 1], [4, 1], [4, 2], [4, 3], [5, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "right", "down", "right", "right", "right", "down", "down", "right", "down", "down", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards1661/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1661/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1661/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1661/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1661/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1661/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1661/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1661/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1661/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1661/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1661/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1661/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1661/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1661/step_13.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1661/step_14.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 2], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[1, 1], [3, 1], [1, 2], [2, 2], [3, 2]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 10], [7, 10], [7, 11], [7, 12], [8, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 15], [10, 16], [11, 16], [11, 17], [12, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 6], [17, 6], [16, 7], [17, 7], [16, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "left", "up", "up", "up", "up", "up", "up", "left", "left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2422/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2422/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2422/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2422/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 12], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[9, 11], [8, 12], [9, 12], [10, 12], [9, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 3], [3, 3], [3, 4], [3, 5], [4, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 15], [16, 15], [14, 16], [15, 16], [16, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[12, 8], [13, 8], [12, 9], [13, 9], [12, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "down"]}], [{"image": "training_data/easy/boards/boards3346/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3346/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3346/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3346/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3346/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 6], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[7, 5], [8, 5], [8, 6], [8, 7], [9, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 11], [12, 11], [13, 11], [12, 12], [12, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 2], [4, 3], [5, 3], [5, 4], [6, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[2, 8], [1, 9], [2, 9], [3, 9], [2, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards1860/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1860/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1860/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1860/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1860/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1860/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1860/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1860/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1860/step_8.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1860/step_9.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 6], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[2, 5], [2, 6], [3, 6], [3, 7], [4, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 9], [12, 9], [10, 10], [11, 10], [12, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 14], [15, 14], [15, 15], [15, 16], [16, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 1], [8, 1], [7, 2], [8, 2], [7, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "left", "up", "left", "left", "up", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards3312/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3312/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3312/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3312/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3312/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3312/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[7, 12], [8, 12], [8, 13], [8, 14], [9, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 1], [11, 2], [12, 2], [13, 2], [12, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 15], [3, 15], [1, 16], [2, 16], [3, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[4, 6], [5, 6], [6, 6], [5, 7], [5, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards25/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards25/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards25/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards25/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards25/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards25/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards25/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards25/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards25/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards25/step_9.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards25/step_10.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards25/step_11.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards25/step_12.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards25/step_13.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 15], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[2, 14], [3, 14], [2, 15], [3, 15], [2, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[8, 5], [10, 5], [8, 6], [9, 6], [10, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 11], [6, 11], [7, 11], [6, 12], [6, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 8], [13, 9], [14, 9], [14, 10], [15, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "down", "left", "left", "down", "left", "down", "down", "down", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards2124/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2124/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2124/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2124/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2124/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2124/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2124/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 3], "target_shape": "W", "target_color": "yellow", "info": [{"piece_grids": [[8, 2], [8, 3], [9, 3], [9, 4], [10, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 14], [12, 14], [12, 15], [12, 16], [13, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 9], [5, 9], [3, 10], [4, 10], [5, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[16, 6], [15, 7], [16, 7], [17, 7], [16, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards827/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards827/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards827/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards827/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards827/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards827/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards827/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards827/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 7], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[3, 6], [4, 6], [5, 6], [4, 7], [4, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 14], [10, 15], [11, 15], [12, 15], [11, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 2], [14, 3], [15, 3], [15, 4], [16, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 11], [9, 11], [8, 12], [9, 12], [8, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "left", "left", "left", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards2495/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2495/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2495/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2495/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 10], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[11, 9], [10, 10], [11, 10], [12, 10], [11, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 1], [6, 1], [6, 2], [6, 3], [7, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 14], [4, 14], [2, 15], [3, 15], [4, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[13, 4], [14, 4], [15, 4], [14, 5], [14, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "down", "right"]}], [{"image": "training_data/easy/boards/boards1333/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1333/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1333/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1333/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1333/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1333/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1333/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1333/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1333/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1333/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 7], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[1, 6], [3, 6], [1, 7], [2, 7], [3, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 15], [14, 15], [13, 16], [14, 16], [13, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 11], [10, 11], [11, 11], [10, 12], [10, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 3], [6, 4], [7, 4], [8, 4], [7, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "left", "up", "left", "left", "up", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1284/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1284/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1284/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1284/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1284/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1284/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1284/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1284/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1284/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1284/step_9.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1284/step_10.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1284/step_11.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1284/step_12.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1284/step_13.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 15], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[1, 14], [3, 14], [1, 15], [2, 15], [3, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 11], [6, 11], [5, 12], [6, 12], [5, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[16, 3], [15, 4], [16, 4], [17, 4], [16, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[7, 7], [8, 7], [8, 8], [8, 9], [9, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "left", "down", "left", "left", "down", "left", "left", "down", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards3187/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3187/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3187/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3187/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3187/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3187/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3187/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3187/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 14], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[6, 13], [7, 13], [7, 14], [7, 15], [8, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 9], [3, 9], [4, 9], [3, 10], [3, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 1], [15, 1], [14, 2], [15, 2], [14, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[10, 4], [12, 4], [10, 5], [11, 5], [12, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "left", "down", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards1715/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1715/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1715/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1715/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 10], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[10, 9], [12, 9], [10, 10], [11, 10], [12, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[6, 1], [5, 2], [6, 2], [7, 2], [6, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 14], [3, 14], [3, 15], [3, 16], [4, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[13, 4], [14, 4], [15, 4], [14, 5], [14, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "down", "right"]}], [{"image": "training_data/easy/boards/boards3010/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3010/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3010/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3010/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3010/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3010/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3010/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3010/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3010/step_8.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3010/step_9.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3010/step_10.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 14], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[3, 13], [4, 13], [4, 14], [4, 15], [5, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[12, 1], [11, 2], [12, 2], [13, 2], [12, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 6], [7, 6], [8, 6], [7, 7], [7, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 10], [15, 11], [16, 11], [16, 12], [17, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "left", "down", "left", "down", "left", "left", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards1989/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1989/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1989/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1989/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1989/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 8], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[11, 7], [11, 8], [12, 8], [12, 9], [13, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 13], [2, 13], [3, 13], [2, 14], [2, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 4], [16, 4], [15, 5], [16, 5], [15, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 10], [10, 10], [8, 11], [9, 11], [10, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1126/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1126/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1126/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1126/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1126/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1126/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1126/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1126/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1126/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1126/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1126/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1126/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1126/step_12.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1126/step_13.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 16], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[14, 15], [15, 15], [16, 15], [15, 16], [15, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 12], [8, 12], [7, 13], [8, 13], [7, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 3], [1, 4], [2, 4], [3, 4], [2, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[9, 9], [9, 10], [10, 10], [10, 11], [11, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "right", "down", "right", "right", "right", "down", "right", "down", "down", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards2355/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2355/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2355/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2355/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2355/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2355/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2355/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2355/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2355/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2355/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 4], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[4, 3], [4, 4], [5, 4], [5, 5], [6, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 7], [14, 7], [14, 8], [14, 9], [15, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 13], [9, 13], [7, 14], [8, 14], [9, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 10], [11, 10], [12, 10], [11, 11], [11, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "left", "up", "up", "left", "left", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards502/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards502/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards502/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards502/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards502/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards502/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards502/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards502/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 9], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[16, 8], [17, 8], [16, 9], [17, 9], [16, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 5], [10, 5], [8, 6], [9, 6], [10, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 11], [12, 12], [13, 12], [13, 13], [14, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [3, 1], [3, 2], [3, 3], [4, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2971/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2971/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2971/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2971/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2971/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2971/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[8, 12], [7, 13], [8, 13], [9, 13], [8, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 9], [4, 9], [3, 10], [4, 10], [3, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 4], [14, 4], [15, 4], [14, 5], [14, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[10, 1], [11, 1], [11, 2], [11, 3], [12, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards2151/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2151/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2151/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 7], "target_shape": "W", "target_color": "yellow", "info": [{"piece_grids": [[8, 6], [8, 7], [9, 7], [9, 8], [10, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 1], [14, 1], [15, 1], [14, 2], [14, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 15], [7, 15], [5, 16], [6, 16], [7, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 12], [2, 13], [3, 13], [4, 13], [3, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up"]}], [{"image": "training_data/easy/boards/boards1914/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1914/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1914/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1914/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1914/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1914/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1914/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1914/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1914/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1914/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1914/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 14], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[13, 13], [13, 14], [14, 14], [14, 15], [15, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[3, 2], [4, 2], [5, 2], [4, 3], [4, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 6], [8, 6], [8, 7], [8, 8], [9, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 9], [10, 10], [11, 10], [12, 10], [11, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "down", "right", "right", "down", "right", "down", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards1659/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1659/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1659/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1659/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1659/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1659/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1659/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1659/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1659/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1659/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 3], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[5, 2], [7, 2], [5, 3], [6, 3], [7, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 9], [14, 9], [14, 10], [14, 11], [15, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[10, 15], [11, 15], [12, 15], [11, 16], [11, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 6], [2, 7], [3, 7], [4, 7], [3, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left", "left", "up", "up", "up", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards2823/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2823/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2823/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2823/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2823/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2823/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2823/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2823/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2823/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2823/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2823/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 2], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[12, 1], [11, 2], [12, 2], [13, 2], [12, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 10], [6, 10], [6, 11], [6, 12], [7, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 15], [3, 15], [2, 16], [3, 16], [2, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 6], [16, 6], [17, 6], [16, 7], [16, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up", "right", "up", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards2441/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2441/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2441/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2441/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2441/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2441/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2441/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2441/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2441/step_8.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2441/step_9.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2441/step_10.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2441/step_11.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2441/step_12.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2441/step_13.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2441/step_14.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 2], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[2, 1], [1, 2], [2, 2], [3, 2], [2, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 10], [8, 10], [6, 11], [7, 11], [8, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 15], [10, 16], [11, 16], [11, 17], [12, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 6], [17, 6], [16, 7], [17, 7], [16, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "up", "left", "up", "left", "left", "up", "up", "left", "left", "up", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards2179/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2179/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2179/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2179/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2179/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2179/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2179/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2179/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2179/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2179/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 3], "target_shape": "W", "target_color": "yellow", "info": [{"piece_grids": [[5, 2], [5, 3], [6, 3], [6, 4], [7, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 9], [14, 9], [14, 10], [14, 11], [15, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[10, 15], [11, 15], [12, 15], [11, 16], [11, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 6], [2, 7], [3, 7], [4, 7], [3, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "up", "left", "up", "up", "up", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards3291/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3291/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3291/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3291/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3291/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3291/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3291/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3291/step_7.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 10], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[2, 9], [3, 9], [3, 10], [3, 11], [4, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 14], [15, 14], [16, 14], [15, 15], [15, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 3], [9, 3], [8, 4], [9, 4], [8, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 6], [11, 7], [12, 7], [12, 8], [13, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "left", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards40/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards40/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 9], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[8, 8], [9, 8], [8, 9], [9, 9], [8, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 13], [4, 13], [2, 14], [3, 14], [4, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[10, 5], [11, 5], [12, 5], [11, 6], [11, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 2], [14, 2], [14, 3], [14, 4], [15, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left"]}], [{"image": "training_data/easy/boards/boards1268/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1268/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1268/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1268/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 10], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[6, 9], [8, 9], [6, 10], [7, 10], [8, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 13], [4, 13], [4, 14], [4, 15], [5, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 1], [14, 2], [15, 2], [15, 3], [16, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 6], [10, 7], [11, 7], [12, 7], [11, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "down"]}], [{"image": "training_data/easy/boards/boards1068/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1068/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1068/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1068/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1068/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1068/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[7, 12], [8, 12], [9, 12], [8, 13], [8, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 5], [3, 5], [2, 6], [3, 6], [2, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 2], [12, 3], [13, 3], [14, 3], [13, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[15, 9], [15, 10], [16, 10], [16, 11], [17, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards2782/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2782/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2782/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2782/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2782/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 6], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[10, 5], [9, 6], [10, 6], [11, 6], [10, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 14], [6, 14], [7, 14], [6, 15], [6, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 9], [15, 9], [14, 10], [15, 10], [14, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 1], [2, 1], [2, 2], [2, 3], [3, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards887/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards887/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards887/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards887/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards887/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards887/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards887/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards887/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards887/step_8.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 6], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[3, 5], [4, 5], [5, 5], [4, 6], [4, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 12], [10, 12], [10, 13], [10, 14], [11, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[6, 8], [8, 8], [6, 9], [7, 9], [8, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 2], [14, 3], [15, 3], [15, 4], [16, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "up", "up", "left", "left", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards2022/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2022/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2022/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 9], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[10, 8], [10, 9], [11, 9], [11, 10], [12, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 4], [2, 4], [2, 5], [2, 6], [3, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 14], [5, 15], [6, 15], [7, 15], [6, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 11], [17, 11], [15, 12], [16, 12], [17, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "right"]}], [{"image": "training_data/easy/boards/boards3039/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3039/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3039/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3039/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3039/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3039/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 4], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[8, 3], [9, 3], [9, 4], [9, 5], [10, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 7], [12, 7], [13, 7], [12, 8], [12, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 10], [1, 11], [2, 11], [2, 12], [3, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[4, 15], [6, 15], [4, 16], [5, 16], [6, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2120/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2120/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2120/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2120/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2120/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2120/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2120/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2120/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2120/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2120/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 6], "target_shape": "W", "target_color": "yellow", "info": [{"piece_grids": [[2, 5], [2, 6], [3, 6], [3, 7], [4, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 9], [12, 9], [10, 10], [11, 10], [12, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 14], [15, 14], [15, 15], [15, 16], [16, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 1], [8, 1], [7, 2], [8, 2], [7, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["up", "left", "left", "left", "left", "up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards20/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards20/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards20/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards20/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards20/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards20/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards20/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards20/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards20/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 13], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[5, 12], [6, 12], [5, 13], [6, 13], [5, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[13, 9], [13, 10], [14, 10], [14, 11], [15, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 6], [9, 6], [7, 7], [8, 7], [9, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 2], [2, 2], [2, 3], [2, 4], [3, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}], "path": ["left", "down", "down", "down", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards1078/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1078/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1078/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1078/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1078/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1078/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1078/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1078/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 4], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[10, 3], [11, 3], [12, 3], [11, 4], [11, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 15], [5, 16], [6, 16], [7, 16], [6, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 11], [3, 11], [3, 12], [3, 13], [4, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 7], [17, 7], [15, 8], [16, 8], [17, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "right", "up", "up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards3046/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3046/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3046/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3046/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 11], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[9, 10], [10, 10], [10, 11], [10, 12], [11, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 14], [4, 14], [3, 15], [4, 15], [3, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 7], [7, 7], [5, 8], [6, 8], [7, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 3], [13, 4], [14, 4], [15, 4], [14, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "down", "down"]}], [{"image": "training_data/easy/boards/boards3234/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3234/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3234/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3234/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3234/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3234/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3234/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3234/step_7.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 5], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[11, 4], [12, 4], [12, 5], [12, 6], [13, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 7], [7, 7], [8, 7], [7, 8], [7, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 10], [17, 10], [16, 11], [17, 11], [16, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 14], [1, 15], [2, 15], [3, 15], [2, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "right", "right", "up", "right", "up", "up"]}], [{"image": "training_data/easy/boards/boards2499/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2499/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2499/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2499/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2499/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2499/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2499/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2499/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2499/step_8.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2499/step_9.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2499/step_10.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2499/step_11.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2499/step_12.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2499/step_13.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 16], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[3, 15], [2, 16], [3, 16], [4, 16], [3, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[11, 7], [12, 7], [13, 7], [12, 8], [12, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 1], [15, 2], [16, 2], [16, 3], [17, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 12], [10, 12], [9, 13], [10, 13], [9, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["down", "down", "left", "left", "down", "left", "down", "down", "down", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards2438/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2438/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2438/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2438/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2438/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2438/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2438/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2438/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2438/step_8.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2438/step_9.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2438/step_10.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2438/step_11.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2438/step_12.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 16], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[14, 15], [13, 16], [14, 16], [15, 16], [14, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 6], [11, 6], [9, 7], [10, 7], [11, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 3], [4, 3], [5, 3], [4, 4], [4, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 12], [7, 12], [7, 13], [7, 14], [8, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["down", "right", "right", "down", "down", "right", "right", "down", "down", "right", "down", "down"]}], [{"image": "training_data/easy/boards/boards829/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards829/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards829/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards829/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards829/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 8], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[5, 7], [6, 7], [7, 7], [6, 8], [6, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 1], [11, 1], [9, 2], [10, 2], [11, 2]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 15], [1, 16], [2, 16], [3, 16], [2, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 12], [15, 12], [14, 13], [15, 13], [14, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards31/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards31/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards31/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards31/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards31/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards31/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 8], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[13, 7], [14, 7], [13, 8], [14, 8], [13, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[4, 13], [5, 13], [5, 14], [5, 15], [6, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 3], [15, 4], [16, 4], [16, 5], [17, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 10], [9, 10], [10, 10], [9, 11], [9, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards1556/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1556/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1556/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1556/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1556/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1556/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1556/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1556/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1556/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 13], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[12, 12], [14, 12], [12, 13], [13, 13], [14, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 4], [9, 4], [9, 5], [9, 6], [10, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 9], [1, 10], [2, 10], [3, 10], [2, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[5, 15], [5, 16], [6, 16], [6, 17], [7, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["down", "right", "down", "right", "right", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards1479/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1479/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1479/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1479/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1479/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1479/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 4], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[8, 3], [10, 3], [8, 4], [9, 4], [10, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 7], [12, 7], [13, 7], [12, 8], [12, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 10], [1, 11], [2, 11], [2, 12], [3, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[4, 15], [5, 15], [5, 16], [5, 17], [6, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards246/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards246/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards246/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards246/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards246/step_4.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards246/step_5.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards246/step_6.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards246/step_7.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards246/step_8.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 4], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[6, 3], [7, 3], [6, 4], [7, 4], [6, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 12], [3, 12], [1, 13], [2, 13], [3, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 9], [14, 10], [15, 10], [15, 11], [16, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[10, 6], [11, 6], [12, 6], [11, 7], [11, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "up", "up", "left", "up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards3163/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3163/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3163/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3163/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3163/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 7], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[10, 6], [11, 6], [11, 7], [11, 8], [12, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 13], [16, 13], [15, 14], [16, 14], [15, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 3], [9, 3], [7, 4], [8, 4], [9, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 9], [2, 10], [3, 10], [3, 11], [4, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards965/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards965/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards965/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards965/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards965/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards965/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards965/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards965/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards965/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 16], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[7, 15], [8, 15], [9, 15], [8, 16], [8, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 8], [11, 8], [11, 9], [11, 10], [12, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 11], [3, 12], [4, 12], [5, 12], [4, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 1], [16, 1], [14, 2], [15, 2], [16, 2]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "down", "left", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards432/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards432/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards432/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards432/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards432/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 7], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[7, 6], [8, 6], [7, 7], [8, 7], [7, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 15], [17, 15], [15, 16], [16, 16], [17, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[4, 11], [3, 12], [4, 12], [5, 12], [4, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 2], [13, 2], [14, 2], [13, 3], [13, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards1115/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1115/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1115/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1115/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1115/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1115/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 7], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[5, 6], [6, 6], [7, 6], [6, 7], [6, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 2], [1, 3], [2, 3], [2, 4], [3, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 10], [17, 10], [15, 11], [16, 11], [17, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[12, 15], [13, 15], [13, 16], [13, 17], [14, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "up", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards1750/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1750/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1750/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1750/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1750/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1750/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1750/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 8], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[3, 7], [5, 7], [3, 8], [4, 8], [5, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 14], [14, 14], [14, 15], [14, 16], [15, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 10], [10, 10], [9, 11], [10, 11], [9, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 10], [7, 11], [8, 11], [8, 12], [9, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "up", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards805/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards805/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards805/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards805/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards805/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards805/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards805/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards805/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 11], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[3, 10], [4, 10], [5, 10], [4, 11], [4, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 13], [15, 13], [15, 14], [15, 15], [16, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 3], [6, 4], [7, 4], [7, 5], [8, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[10, 6], [11, 6], [10, 7], [11, 7], [10, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "left", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards2556/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2556/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2556/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2556/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2556/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2556/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2556/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2556/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2556/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2556/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2556/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 16], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[6, 15], [5, 16], [6, 16], [7, 16], [6, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 10], [2, 11], [3, 11], [3, 12], [4, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 7], [10, 7], [11, 7], [10, 8], [10, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 3], [13, 3], [13, 4], [13, 5], [14, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "down", "down", "down", "down", "left", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards2952/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2952/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2952/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2952/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2952/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2952/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2952/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2952/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2952/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2952/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2952/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 5], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[15, 4], [14, 5], [15, 5], [16, 5], [15, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 11], [3, 11], [2, 12], [3, 12], [2, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 7], [8, 7], [9, 7], [8, 8], [8, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 14], [4, 15], [5, 15], [5, 16], [6, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "right", "right", "right", "up", "up", "right", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards2093/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2093/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2093/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2093/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2093/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2093/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2093/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 15], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[8, 14], [8, 15], [9, 15], [9, 16], [10, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 1], [5, 1], [4, 2], [5, 2], [4, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 8], [14, 8], [15, 8], [14, 9], [14, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 5], [4, 5], [2, 6], [3, 6], [4, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards3496/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3496/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3496/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3496/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3496/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3496/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3496/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3496/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 10], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[14, 9], [15, 9], [15, 10], [15, 11], [16, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[3, 5], [5, 5], [3, 6], [4, 6], [5, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 1], [10, 1], [9, 2], [10, 2], [9, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 14], [11, 15], [12, 15], [13, 15], [12, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "down", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards736/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards736/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards736/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards736/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards736/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards736/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards736/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards736/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards736/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards736/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards736/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 16], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[5, 15], [6, 15], [7, 15], [6, 16], [6, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 10], [2, 11], [3, 11], [3, 12], [4, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 7], [9, 8], [10, 8], [11, 8], [10, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 3], [13, 3], [13, 4], [13, 5], [14, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "down", "down", "down", "down", "left", "left", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards3319/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3319/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3319/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3319/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3319/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3319/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3319/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3319/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3319/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3319/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3319/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3319/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3319/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 14], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[1, 13], [2, 13], [2, 14], [2, 15], [3, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 5], [7, 5], [5, 6], [6, 6], [7, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 1], [15, 2], [16, 2], [17, 2], [16, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 8], [12, 8], [11, 9], [12, 9], [11, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "left", "down", "left", "down", "left", "left", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards1623/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1623/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1623/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1623/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1623/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1623/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1623/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1623/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1623/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1623/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 5], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[3, 4], [5, 4], [3, 5], [4, 5], [5, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[8, 13], [9, 13], [9, 14], [9, 15], [10, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 7], [12, 7], [13, 7], [12, 8], [12, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 1], [14, 2], [15, 2], [15, 3], [16, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "up", "left", "left", "up", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards2901/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2901/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2901/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2901/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2901/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2901/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2901/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2901/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2901/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2901/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 16], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[7, 15], [6, 16], [7, 16], [8, 16], [7, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [7, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 6], [4, 6], [3, 7], [4, 7], [3, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 12], [15, 12], [13, 13], [14, 13], [15, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 1], [10, 1], [11, 1], [10, 2], [10, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "down", "down", "down", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards1824/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1824/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1824/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1824/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1824/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1824/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1824/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1824/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1824/step_8.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1824/step_9.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1824/step_10.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1824/step_11.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1824/step_12.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1824/step_13.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 2], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[14, 1], [14, 2], [15, 2], [15, 3], [16, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 12], [4, 12], [2, 13], [3, 13], [4, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[6, 4], [7, 4], [6, 5], [7, 5], [6, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[8, 15], [9, 15], [10, 15], [9, 16], [9, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right", "up", "right", "up", "up", "up", "right", "right", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards3494/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3494/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3494/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3494/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3494/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3494/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3494/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3494/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 5], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[11, 4], [12, 4], [12, 5], [12, 6], [13, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 7], [7, 7], [8, 7], [7, 8], [7, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 10], [17, 10], [16, 11], [17, 11], [16, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 14], [1, 15], [2, 15], [3, 15], [2, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "up", "up", "right", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards434/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards434/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards434/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards434/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards434/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 12], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[10, 11], [11, 11], [10, 12], [11, 12], [10, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[6, 3], [6, 4], [7, 4], [7, 5], [8, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 7], [15, 7], [15, 8], [15, 9], [16, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[4, 14], [3, 15], [4, 15], [5, 15], [4, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1823/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1823/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1823/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1823/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1823/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1823/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1823/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1823/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1823/step_8.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1823/step_9.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1823/step_10.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1823/step_11.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1823/step_12.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 14], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[1, 13], [1, 14], [2, 14], [2, 15], [3, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[6, 5], [5, 6], [6, 6], [7, 6], [6, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 10], [10, 10], [9, 11], [10, 11], [9, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 2], [15, 2], [13, 3], [14, 3], [15, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "down", "left", "down", "down", "left", "left", "down", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards1526/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1526/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1526/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1526/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1526/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 6], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[7, 5], [9, 5], [7, 6], [8, 6], [9, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 11], [12, 11], [13, 11], [12, 12], [12, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 2], [4, 3], [5, 3], [5, 4], [6, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[2, 8], [1, 9], [2, 9], [3, 9], [2, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards2234/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2234/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2234/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2234/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 8], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[10, 7], [10, 8], [11, 8], [11, 9], [12, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 1], [8, 1], [7, 2], [8, 2], [7, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 15], [16, 15], [17, 15], [16, 16], [16, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[1, 10], [2, 10], [2, 11], [2, 12], [3, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "right", "right"]}], [{"image": "training_data/easy/boards/boards1532/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1532/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1532/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1532/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1532/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1532/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1532/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1532/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 9], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[15, 8], [17, 8], [15, 9], [16, 9], [17, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [2, 2], [3, 2], [3, 3], [4, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 15], [8, 15], [8, 16], [8, 17], [9, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 4], [13, 4], [14, 4], [13, 5], [13, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards526/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards526/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards526/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards526/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards526/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards526/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards526/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards526/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards526/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards526/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 6], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[15, 5], [16, 5], [15, 6], [16, 6], [15, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 14], [7, 15], [8, 15], [8, 16], [9, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 8], [3, 8], [1, 9], [2, 9], [3, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[4, 11], [5, 11], [6, 11], [5, 12], [5, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "up", "right", "right", "up", "right", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards1220/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1220/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1220/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1220/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1220/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1220/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1220/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1220/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 5], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[11, 4], [13, 4], [11, 5], [12, 5], [13, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[9, 13], [8, 14], [9, 14], [10, 14], [9, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 9], [3, 9], [2, 10], [3, 10], [2, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 1], [15, 1], [16, 1], [15, 2], [15, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "up", "right", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards585/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards585/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards585/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards585/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards585/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards585/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards585/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards585/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards585/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards585/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards585/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards585/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards585/step_12.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 14], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[16, 13], [17, 13], [16, 14], [17, 14], [16, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[4, 10], [5, 10], [6, 10], [5, 11], [5, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 1], [9, 2], [10, 2], [11, 2], [10, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 6], [13, 6], [13, 7], [13, 8], [14, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "down", "right", "right", "right", "right", "right", "down", "down", "right", "down", "down"]}], [{"image": "training_data/easy/boards/boards713/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards713/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards713/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards713/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards713/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards713/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards713/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 5], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[6, 4], [7, 4], [8, 4], [7, 5], [7, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 9], [13, 9], [11, 10], [12, 10], [13, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 1], [4, 1], [4, 2], [4, 3], [5, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 12], [14, 13], [15, 13], [16, 13], [15, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "up", "up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards1613/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1613/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1613/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1613/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1613/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 12], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[7, 11], [9, 11], [7, 12], [8, 12], [9, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 7], [2, 7], [2, 8], [2, 9], [3, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[5, 1], [6, 1], [5, 2], [6, 2], [5, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 4], [13, 4], [14, 4], [13, 5], [13, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards1906/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1906/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1906/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1906/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1906/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1906/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1906/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1906/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1906/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1906/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1906/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1906/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1906/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1906/step_13.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 16], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[14, 15], [14, 16], [15, 16], [15, 17], [16, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 12], [8, 12], [7, 13], [8, 13], [7, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 3], [2, 3], [3, 3], [2, 4], [2, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[9, 9], [11, 9], [9, 10], [10, 10], [11, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "down", "right", "down", "down", "right", "down", "right", "down", "right", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards3055/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3055/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3055/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 8], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[9, 7], [10, 7], [10, 8], [10, 9], [11, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 1], [4, 2], [5, 2], [5, 3], [6, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 15], [15, 15], [13, 16], [14, 16], [15, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[2, 4], [3, 4], [2, 5], [3, 5], [2, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "up"]}], [{"image": "training_data/easy/boards/boards3008/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3008/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3008/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3008/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3008/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3008/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3008/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3008/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3008/step_8.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3008/step_9.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 14], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[4, 13], [5, 13], [5, 14], [5, 15], [6, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[13, 4], [12, 5], [13, 5], [14, 5], [13, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 7], [2, 7], [3, 7], [2, 8], [2, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 10], [10, 10], [8, 11], [9, 11], [10, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "left", "left", "left", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1819/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1819/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1819/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1819/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1819/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1819/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1819/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1819/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1819/step_8.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 2], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[9, 1], [9, 2], [10, 2], [10, 3], [11, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 7], [6, 7], [5, 8], [6, 8], [5, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 4], [15, 4], [15, 5], [15, 6], [16, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 12], [3, 12], [1, 13], [2, 13], [3, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up", "up", "right", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2683/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2683/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2683/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2683/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2683/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2683/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2683/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2683/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 6], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[13, 5], [12, 6], [13, 6], [14, 6], [13, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 13], [8, 13], [6, 14], [7, 14], [8, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 2], [17, 2], [16, 3], [17, 3], [16, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 9], [4, 9], [4, 10], [4, 11], [5, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "right", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards1213/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1213/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1213/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1213/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1213/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1213/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1213/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1213/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1213/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 4], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[5, 3], [7, 3], [5, 4], [6, 4], [7, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 14], [14, 14], [14, 15], [14, 16], [15, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[1, 8], [2, 8], [3, 8], [2, 9], [2, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 11], [10, 11], [9, 12], [10, 12], [9, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "up", "left", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards91/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards91/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards91/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards91/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards91/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards91/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards91/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards91/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 10], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[3, 9], [4, 9], [3, 10], [4, 10], [3, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 13], [12, 13], [10, 14], [11, 14], [12, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[7, 1], [6, 2], [7, 2], [8, 2], [7, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 5], [16, 5], [16, 6], [16, 7], [17, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "left", "left", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards1123/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1123/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1123/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1123/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1123/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1123/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1123/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1123/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 6], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[12, 5], [13, 5], [14, 5], [13, 6], [13, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 13], [8, 13], [6, 14], [7, 14], [8, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 2], [17, 2], [16, 3], [17, 3], [16, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 9], [4, 9], [4, 10], [4, 11], [5, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "up", "up", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards417/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards417/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards417/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards417/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards417/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards417/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards417/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards417/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards417/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards417/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards417/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 16], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[6, 15], [7, 15], [6, 16], [7, 16], [6, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 6], [1, 7], [2, 7], [3, 7], [2, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 11], [12, 11], [13, 11], [12, 12], [12, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "left", "left", "down", "down", "down", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards1158/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1158/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1158/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1158/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1158/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1158/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1158/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1158/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 13], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[5, 12], [6, 12], [7, 12], [6, 13], [6, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[9, 15], [9, 16], [10, 16], [10, 17], [11, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 7], [15, 7], [14, 8], [15, 8], [14, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 1], [2, 1], [2, 2], [2, 3], [3, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}], "path": ["left", "down", "left", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards1497/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1497/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1497/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1497/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1497/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1497/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 13], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[9, 12], [11, 12], [9, 13], [10, 13], [11, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[5, 5], [6, 5], [7, 5], [6, 6], [6, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 9], [2, 10], [3, 10], [3, 11], [4, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 2], [14, 2], [13, 3], [14, 3], [13, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "down", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards2973/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2973/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2973/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2973/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2973/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2973/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2973/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2973/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2973/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2973/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2973/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2973/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 15], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[4, 14], [3, 15], [4, 15], [5, 15], [4, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[11, 7], [11, 8], [12, 8], [12, 9], [13, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 3], [8, 3], [9, 3], [8, 4], [8, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 11], [17, 11], [15, 12], [16, 12], [17, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "down", "down", "down", "down", "left", "down", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards295/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards295/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards295/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards295/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards295/step_4.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards295/step_5.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards295/step_6.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards295/step_7.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 9], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[2, 8], [3, 8], [2, 9], [3, 9], [2, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 13], [10, 13], [10, 14], [10, 15], [11, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 4], [14, 5], [15, 5], [16, 5], [15, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[6, 1], [7, 1], [8, 1], [7, 2], [7, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "left", "left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2358/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2358/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 9], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[7, 8], [7, 9], [8, 9], [8, 10], [9, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 14], [3, 14], [3, 15], [3, 16], [4, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[11, 3], [12, 3], [11, 4], [12, 4], [11, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[16, 11], [15, 12], [16, 12], [17, 12], [16, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["left"]}], [{"image": "training_data/easy/boards/boards1453/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1453/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1453/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1453/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1453/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1453/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1453/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1453/step_7.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1453/step_8.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1453/step_9.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1453/step_10.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1453/step_11.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1453/step_12.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1453/step_13.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 16], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[14, 15], [16, 15], [14, 16], [15, 16], [16, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 8], [10, 8], [9, 9], [10, 9], [9, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 11], [11, 12], [12, 12], [12, 13], [13, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 4], [2, 4], [3, 4], [2, 5], [2, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "down", "right", "down", "down", "right", "right", "right", "right", "down", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards547/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards547/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards547/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards547/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards547/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards547/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards547/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards547/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 5], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[6, 4], [7, 4], [6, 5], [7, 5], [6, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 13], [10, 13], [11, 13], [10, 14], [10, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "up", "up", "left", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards1931/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1931/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1931/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1931/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1931/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1931/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[7, 12], [7, 13], [8, 13], [8, 14], [9, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 9], [4, 9], [3, 10], [4, 10], [3, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 4], [14, 4], [15, 4], [14, 5], [14, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 1], [10, 2], [11, 2], [12, 2], [11, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards1493/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1493/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1493/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1493/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1493/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1493/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1493/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 5], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[6, 4], [8, 4], [6, 5], [7, 5], [8, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 9], [12, 9], [12, 10], [12, 11], [13, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 1], [3, 2], [4, 2], [5, 2], [4, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 12], [15, 12], [16, 12], [15, 13], [15, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "left", "left", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2727/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2727/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2727/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2727/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2727/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2727/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2727/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2727/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2727/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2727/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2727/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2727/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 13], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[16, 12], [15, 13], [16, 13], [17, 13], [16, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 9], [7, 9], [5, 10], [6, 10], [7, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 3], [13, 3], [13, 4], [13, 5], [14, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 6], [2, 7], [3, 7], [3, 8], [4, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "down", "right", "down", "down", "right", "right", "right", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards2362/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2362/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2362/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2362/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2362/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2362/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2362/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2362/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 13], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[5, 12], [5, 13], [6, 13], [6, 14], [7, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 3], [16, 3], [17, 3], [16, 4], [16, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 6], [2, 6], [2, 7], [2, 8], [3, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 9], [13, 9], [12, 10], [13, 10], [12, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "left", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards3356/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3356/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3356/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3356/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3356/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3356/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3356/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3356/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3356/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 6], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[13, 5], [14, 5], [14, 6], [14, 7], [15, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 10], [6, 10], [7, 10], [6, 11], [6, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 14], [4, 14], [2, 15], [3, 15], [4, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[10, 2], [11, 2], [10, 3], [11, 3], [10, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "up", "right", "up", "right", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards3067/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3067/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3067/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3067/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3067/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3067/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3067/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3067/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3067/step_8.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3067/step_9.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3067/step_10.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3067/step_11.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3067/step_12.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3067/step_13.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 16], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[14, 15], [15, 15], [15, 16], [15, 17], [16, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 3], [11, 3], [9, 4], [10, 4], [11, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 10], [6, 10], [7, 10], [6, 11], [6, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 6], [1, 7], [2, 7], [2, 8], [3, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "down", "right", "down", "down", "right", "right", "down", "down", "down", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards2974/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2974/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2974/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2974/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2974/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2974/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2974/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2974/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 5], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[12, 4], [11, 5], [12, 5], [13, 5], [12, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 7], [7, 7], [8, 7], [7, 8], [7, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 10], [17, 10], [16, 11], [17, 11], [16, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 14], [2, 14], [2, 15], [2, 16], [3, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "right", "up", "up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards951/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards951/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards951/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards951/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards951/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards951/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards951/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards951/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 10], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[2, 9], [3, 9], [4, 9], [3, 10], [3, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 14], [14, 15], [15, 15], [16, 15], [15, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 3], [9, 3], [8, 4], [9, 4], [8, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 6], [11, 7], [12, 7], [12, 8], [13, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards3125/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3125/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3125/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3125/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3125/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3125/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 4], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[8, 3], [9, 3], [9, 4], [9, 5], [10, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 14], [4, 15], [5, 15], [6, 15], [5, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[2, 6], [3, 6], [2, 7], [3, 7], [2, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 11], [14, 11], [15, 11], [14, 12], [14, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2960/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2960/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2960/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2960/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2960/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 6], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[8, 5], [7, 6], [8, 6], [9, 6], [8, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 10], [12, 10], [12, 11], [12, 12], [13, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 15], [5, 15], [3, 16], [4, 16], [5, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 1], [15, 1], [16, 1], [15, 2], [15, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["up", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards2327/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2327/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2327/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2327/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2327/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2327/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2327/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2327/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2327/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2327/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2327/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2327/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2327/step_12.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 2], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[3, 1], [3, 2], [4, 2], [4, 3], [5, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 14], [8, 14], [8, 15], [8, 16], [9, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 9], [17, 9], [15, 10], [16, 10], [17, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[10, 5], [11, 5], [12, 5], [11, 6], [11, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "up", "up", "up", "up", "left", "up", "up", "left", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards2933/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2933/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 10], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[9, 9], [8, 10], [9, 10], [10, 10], [9, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 15], [16, 15], [14, 16], [15, 16], [16, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[4, 12], [5, 12], [5, 13], [5, 14], [6, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[11, 1], [11, 2], [12, 2], [12, 3], [13, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["down"]}], [{"image": "training_data/easy/boards/boards353/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards353/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards353/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards353/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards353/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards353/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards353/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards353/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 13], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[6, 12], [7, 12], [6, 13], [7, 13], [6, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 4], [2, 5], [3, 5], [3, 6], [4, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 8], [13, 8], [14, 8], [13, 9], [13, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 1], [8, 2], [9, 2], [10, 2], [9, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "left", "left", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards1341/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1341/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1341/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1341/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1341/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1341/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1341/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1341/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1341/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1341/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 16], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[6, 15], [8, 15], [6, 16], [7, 16], [8, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 6], [4, 6], [3, 7], [4, 7], [3, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 12], [14, 12], [14, 13], [14, 14], [15, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 1], [10, 1], [11, 1], [10, 2], [10, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "down", "down", "down", "left", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards3568/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3568/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3568/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3568/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3568/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3568/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3568/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3568/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3568/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3568/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3568/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3568/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 13], "target_shape": "Z", "target_color": "magenta", "info": [{"piece_grids": [[15, 12], [16, 12], [16, 13], [16, 14], [17, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 6], [7, 6], [6, 7], [7, 7], [6, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 15], [10, 16], [11, 16], [11, 17], [12, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 2], [1, 3], [2, 3], [3, 3], [2, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "right", "right", "right", "down", "right", "right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards894/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards894/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards894/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards894/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards894/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards894/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards894/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards894/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 5], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[11, 4], [12, 4], [13, 4], [12, 5], [12, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[7, 7], [6, 8], [7, 8], [8, 8], [7, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 10], [17, 10], [16, 11], [17, 11], [16, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 14], [2, 14], [2, 15], [2, 16], [3, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up", "right", "right", "right", "up", "up"]}], [{"image": "training_data/easy/boards/boards1719/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1719/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1719/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1719/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1719/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1719/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1719/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1719/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1719/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1719/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1719/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1719/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1719/step_12.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1719/step_13.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 16], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[2, 15], [4, 15], [2, 16], [3, 16], [4, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[11, 7], [12, 7], [13, 7], [12, 8], [12, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 1], [15, 2], [16, 2], [16, 3], [17, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 12], [10, 12], [9, 13], [10, 13], [9, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["down", "left", "down", "left", "left", "down", "down", "down", "down", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards3081/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3081/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3081/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 10], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[7, 9], [8, 9], [8, 10], [8, 11], [9, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 6], [10, 7], [11, 7], [12, 7], [11, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 3], [13, 4], [14, 4], [14, 5], [15, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 12], [4, 12], [3, 13], [4, 13], [3, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["down", "left"]}], [{"image": "training_data/easy/boards/boards2011/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2011/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2011/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2011/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2011/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2011/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2011/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2011/step_7.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 3], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[7, 2], [7, 3], [8, 3], [8, 4], [9, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 12], [16, 12], [14, 13], [15, 13], [16, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 9], [12, 9], [12, 10], [12, 11], [13, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 6], [4, 6], [3, 7], [4, 7], [3, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left", "up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2740/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2740/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2740/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 10], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[10, 9], [9, 10], [10, 10], [11, 10], [10, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 12], [3, 13], [4, 13], [4, 14], [5, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[16, 5], [17, 5], [16, 6], [17, 6], [16, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 1], [7, 1], [7, 2], [7, 3], [8, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "down"]}], [{"image": "training_data/easy/boards/boards258/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards258/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards258/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards258/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards258/step_4.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards258/step_5.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards258/step_6.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards258/step_7.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards258/step_8.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards258/step_9.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 14], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[13, 13], [14, 13], [13, 14], [14, 14], [13, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 7], [7, 7], [7, 8], [7, 9], [8, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 10], [17, 10], [15, 11], [16, 11], [17, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 1], [9, 2], [10, 2], [10, 3], [11, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "down", "right", "down", "down", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards2839/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2839/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2839/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2839/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2839/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2839/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2839/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2839/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 4], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[11, 3], [10, 4], [11, 4], [12, 4], [11, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 6], [3, 6], [2, 7], [3, 7], [2, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[6, 10], [6, 11], [7, 11], [7, 12], [8, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 13], [16, 13], [16, 14], [16, 15], [17, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["right", "up", "up", "up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards450/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards450/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards450/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards450/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards450/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards450/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards450/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 8], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[4, 7], [5, 7], [4, 8], [5, 8], [4, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 14], [15, 14], [13, 15], [14, 15], [15, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 10], [9, 10], [10, 10], [9, 11], [9, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 10], [7, 11], [8, 11], [8, 12], [9, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "up", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards3147/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3147/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3147/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3147/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3147/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3147/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3147/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3147/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 5], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[5, 4], [6, 4], [6, 5], [6, 6], [7, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 13], [11, 13], [10, 14], [11, 14], [10, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 1], [15, 2], [16, 2], [17, 2], [16, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["up", "left", "left", "left", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards899/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards899/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards899/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards899/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards899/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards899/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards899/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards899/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards899/step_8.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 5], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[12, 4], [13, 4], [14, 4], [13, 5], [13, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[6, 10], [5, 11], [6, 11], [7, 11], [6, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 1], [10, 1], [9, 2], [10, 2], [9, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 13], [17, 13], [15, 14], [16, 14], [17, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["right", "right", "right", "right", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2475/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2475/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2475/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2475/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2475/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2475/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2475/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2475/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 3], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[10, 2], [9, 3], [10, 3], [11, 3], [10, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 10], [2, 10], [2, 11], [2, 12], [3, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 6], [15, 6], [14, 7], [15, 7], [14, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[4, 13], [5, 13], [6, 13], [5, 14], [5, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "up", "up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2700/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2700/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2700/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2700/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2700/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 6], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[8, 5], [7, 6], [8, 6], [9, 6], [8, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 10], [12, 10], [12, 11], [12, 12], [13, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 15], [5, 15], [3, 16], [4, 16], [5, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 1], [15, 1], [16, 1], [15, 2], [15, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["up", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards699/step_0.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards699/step_1.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards699/step_2.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards699/step_3.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards699/step_4.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards699/step_5.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 4], "target_shape": "T", "target_color": "red", "info": [{"piece_grids": [[8, 3], [9, 3], [10, 3], [9, 4], [9, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 7], [11, 8], [12, 8], [13, 8], [12, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 10], [1, 11], [2, 11], [2, 12], [3, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[4, 15], [6, 15], [4, 16], [5, 16], [6, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards3418/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3418/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3418/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3418/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3418/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3418/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3418/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3418/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 4], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[10, 3], [11, 3], [11, 4], [11, 5], [12, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 15], [6, 15], [7, 15], [6, 16], [6, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 11], [2, 12], [3, 12], [4, 12], [3, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 7], [17, 7], [15, 8], [16, 8], [17, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "right", "right", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards890/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards890/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards890/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards890/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards890/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards890/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards890/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards890/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards890/step_8.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards890/step_9.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 3], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[11, 2], [12, 2], [13, 2], [12, 3], [12, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 15], [4, 15], [2, 16], [3, 16], [4, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[7, 11], [8, 11], [8, 12], [8, 13], [9, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 8], [15, 9], [16, 9], [17, 9], [16, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "right", "up", "up", "right", "up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards2834/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2834/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2834/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2834/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2834/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2834/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2834/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2834/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2834/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 7], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[3, 6], [2, 7], [3, 7], [4, 7], [3, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[5, 3], [6, 3], [6, 4], [6, 5], [7, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[10, 12], [11, 12], [10, 13], [11, 13], [10, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 15], [13, 15], [14, 15], [13, 16], [13, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "left", "left", "left", "up", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards2632/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2632/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2632/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2632/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2632/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2632/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2632/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2632/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2632/step_8.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 11], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[3, 10], [2, 11], [3, 11], [4, 11], [3, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 2], [12, 2], [12, 3], [12, 4], [13, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[16, 13], [17, 13], [16, 14], [17, 14], [16, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 6], [7, 6], [8, 6], [7, 7], [7, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "left", "left", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards985/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards985/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards985/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards985/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards985/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards985/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards985/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards985/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards985/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards985/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards985/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards985/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 4], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[2, 3], [3, 3], [4, 3], [3, 4], [3, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[16, 7], [15, 8], [16, 8], [17, 8], [16, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 14], [8, 14], [8, 15], [8, 16], [9, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 11], [13, 11], [12, 12], [13, 12], [12, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "up", "up", "left", "up", "up", "left", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards1292/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1292/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1292/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1292/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1292/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1292/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1292/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1292/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1292/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 6], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[13, 5], [15, 5], [13, 6], [14, 6], [15, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 11], [5, 12], [6, 12], [6, 13], [7, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 8], [2, 9], [3, 9], [4, 9], [3, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 2], [11, 2], [10, 3], [11, 3], [10, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "right", "right", "right", "up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards623/step_0.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards623/step_1.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards623/step_2.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards623/step_3.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards623/step_4.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards623/step_5.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards623/step_6.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards623/step_7.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards623/step_8.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards623/step_9.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards623/step_10.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards623/step_11.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 13], "target_shape": "T", "target_color": "red", "info": [{"piece_grids": [[15, 12], [16, 12], [17, 12], [16, 13], [16, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 1], [11, 2], [12, 2], [12, 3], [13, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[4, 7], [6, 7], [4, 8], [5, 8], [6, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 15], [9, 15], [8, 16], [9, 16], [8, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right", "right", "right", "down", "down", "down", "right", "right", "right", "down"]}], [{"image": "training_data/easy/boards/boards2789/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2789/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2789/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 9], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[11, 8], [10, 9], [11, 9], [12, 9], [11, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 12], [14, 12], [14, 13], [14, 14], [15, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[2, 5], [3, 5], [4, 5], [3, 6], [3, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[6, 15], [7, 15], [6, 16], [7, 16], [6, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right"]}], [{"image": "training_data/easy/boards/boards1934/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1934/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1934/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1934/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1934/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1934/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1934/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1934/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 5], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[11, 4], [11, 5], [12, 5], [12, 6], [13, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 7], [7, 7], [8, 7], [7, 8], [7, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 10], [17, 10], [16, 11], [17, 11], [16, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 14], [1, 15], [2, 15], [3, 15], [2, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "right", "up", "up", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards2049/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2049/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2049/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2049/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2049/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2049/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2049/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2049/step_7.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2049/step_8.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 14], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[5, 13], [5, 14], [6, 14], [6, 15], [7, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 7], [15, 7], [15, 8], [15, 9], [16, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[3, 4], [4, 4], [3, 5], [4, 5], [3, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[10, 10], [12, 10], [10, 11], [11, 11], [12, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "down", "down", "left", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards3278/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3278/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3278/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3278/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3278/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3278/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3278/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3278/step_7.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3278/step_8.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3278/step_9.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3278/step_10.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3278/step_11.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 16], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[12, 15], [13, 15], [13, 16], [13, 17], [14, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[15, 4], [15, 5], [16, 5], [16, 6], [17, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 10], [4, 11], [5, 11], [6, 11], [5, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[1, 1], [3, 1], [1, 2], [2, 2], [3, 2]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "down", "down", "down", "down", "right", "right", "down", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards2751/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2751/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2751/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2751/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2751/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2751/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2751/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2751/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2751/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2751/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2751/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 4], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[4, 3], [3, 4], [4, 4], [5, 4], [4, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[9, 11], [11, 11], [9, 12], [10, 12], [11, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 7], [14, 7], [13, 8], [14, 8], [13, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[15, 14], [15, 15], [16, 15], [16, 16], [17, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "left", "left", "up", "up", "up", "up", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards3168/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3168/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3168/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 11], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[8, 10], [9, 10], [9, 11], [9, 12], [10, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[6, 15], [7, 15], [6, 16], [7, 16], [6, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 4], [11, 5], [12, 5], [13, 5], [12, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 7], [3, 7], [4, 7], [3, 8], [3, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down"]}], [{"image": "training_data/easy/boards/boards2540/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2540/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2540/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2540/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2540/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2540/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2540/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2540/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 8], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[15, 7], [14, 8], [15, 8], [16, 8], [15, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 12], [8, 12], [6, 13], [7, 13], [8, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 4], [2, 4], [3, 4], [2, 5], [2, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[9, 1], [9, 2], [10, 2], [10, 3], [11, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "right", "right", "right", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards592/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards592/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards592/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards592/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards592/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards592/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 5], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[8, 4], [9, 4], [8, 5], [9, 5], [8, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[10, 15], [11, 15], [11, 16], [11, 17], [12, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 7], [14, 8], [15, 8], [15, 9], [16, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 11], [1, 12], [2, 12], [3, 12], [2, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2427/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2427/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2427/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2427/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2427/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2427/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2427/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2427/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2427/step_8.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2427/step_9.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2427/step_10.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2427/step_11.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 2], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[5, 1], [4, 2], [5, 2], [6, 2], [5, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[1, 8], [2, 8], [3, 8], [2, 9], [2, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 15], [14, 15], [13, 16], [14, 16], [13, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[15, 4], [17, 4], [15, 5], [16, 5], [17, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "left", "left", "up", "up", "up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2276/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2276/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2276/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2276/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2276/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2276/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2276/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2276/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2276/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2276/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2276/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 12], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[1, 11], [1, 12], [2, 12], [2, 13], [3, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 15], [12, 15], [12, 16], [12, 17], [13, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 8], [5, 8], [6, 8], [5, 9], [5, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [5, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 4], [9, 4], [8, 5], [9, 5], [8, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "down", "left", "left", "left", "left", "down", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards2769/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2769/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2769/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2769/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2769/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 8], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[12, 7], [11, 8], [12, 8], [13, 8], [12, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 13], [2, 13], [3, 13], [2, 14], [2, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 4], [16, 4], [15, 5], [16, 5], [15, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 10], [8, 11], [9, 11], [9, 12], [10, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards419/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards419/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards419/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards419/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards419/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards419/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards419/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards419/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards419/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards419/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards419/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards419/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards419/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards419/step_13.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 16], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[3, 15], [4, 15], [3, 16], [4, 16], [3, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[12, 7], [11, 8], [12, 8], [13, 8], [12, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 1], [15, 2], [16, 2], [16, 3], [17, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 12], [9, 12], [10, 12], [9, 13], [9, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["left", "left", "left", "down", "left", "down", "down", "left", "down", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards548/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards548/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards548/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards548/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards548/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards548/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[8, 12], [9, 12], [8, 13], [9, 13], [8, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 5], [2, 5], [3, 5], [2, 6], [2, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 2], [12, 3], [13, 3], [14, 3], [13, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[15, 9], [15, 10], [16, 10], [16, 11], [17, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards2968/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2968/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2968/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2968/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2968/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2968/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 11], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[12, 10], [11, 11], [12, 11], [13, 11], [12, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 15], [7, 16], [8, 16], [8, 17], [9, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 7], [16, 7], [14, 8], [15, 8], [16, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[4, 1], [5, 1], [6, 1], [5, 2], [5, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "down", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards924/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards924/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards924/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards924/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards924/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards924/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards924/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards924/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards924/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards924/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards924/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 5], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[14, 4], [15, 4], [16, 4], [15, 5], [15, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 8], [4, 8], [4, 9], [4, 10], [5, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 11], [8, 11], [7, 12], [8, 12], [7, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 15], [9, 16], [10, 16], [10, 17], [11, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right", "right", "right", "up", "right", "up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards901/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards901/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards901/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards901/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards901/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards901/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards901/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards901/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards901/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards901/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards901/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 6], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[1, 5], [2, 5], [3, 5], [2, 6], [2, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[16, 12], [17, 12], [16, 13], [17, 13], [16, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 2], [9, 2], [9, 3], [9, 4], [10, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 8], [4, 9], [5, 9], [6, 9], [5, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["left", "left", "up", "up", "left", "left", "up", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2761/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2761/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2761/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2761/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2761/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2761/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2761/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2761/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2761/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2761/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2761/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2761/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2761/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2761/step_13.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2761/step_14.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 16], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[2, 15], [1, 16], [2, 16], [3, 16], [2, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 12], [6, 12], [7, 12], [6, 13], [6, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 8], [14, 8], [13, 9], [14, 9], [13, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[15, 1], [17, 1], [15, 2], [16, 2], [17, 2]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "left", "left", "down", "left", "down", "left", "left", "down", "down", "left", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards1916/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1916/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1916/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1916/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1916/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1916/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1916/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 5], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[6, 4], [6, 5], [7, 5], [7, 6], [8, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[10, 10], [11, 10], [11, 11], [11, 12], [12, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 7], [3, 8], [4, 8], [5, 8], [4, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 13], [17, 13], [15, 14], [16, 14], [17, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "up", "up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards1475/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1475/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1475/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1475/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1475/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1475/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1475/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1475/step_7.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1475/step_8.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1475/step_9.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1475/step_10.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1475/step_11.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 5], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[15, 4], [17, 4], [15, 5], [16, 5], [17, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[4, 7], [3, 8], [4, 8], [5, 8], [4, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[6, 14], [7, 14], [8, 14], [7, 15], [7, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 10], [11, 10], [10, 11], [11, 11], [10, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "up", "right", "right", "right", "right", "up", "up", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards1388/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1388/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1388/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1388/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1388/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1388/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1388/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1388/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1388/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 14], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[11, 13], [13, 13], [11, 14], [12, 14], [13, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[9, 10], [10, 10], [9, 11], [10, 11], [9, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 5], [15, 6], [16, 6], [16, 7], [17, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 1], [6, 1], [7, 1], [6, 2], [6, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "right", "right", "down", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards1721/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1721/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1721/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1721/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1721/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1721/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1721/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1721/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1721/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1721/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1721/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1721/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1721/step_12.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1721/step_13.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1721/step_14.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 16], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[1, 15], [3, 15], [1, 16], [2, 16], [3, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 12], [6, 12], [7, 12], [6, 13], [6, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 8], [14, 8], [13, 9], [14, 9], [13, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "down", "down", "down", "down", "down", "left", "left", "left", "left", "down", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards1551/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1551/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1551/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1551/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1551/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1551/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1551/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1551/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1551/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1551/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1551/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 4], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[3, 3], [5, 3], [3, 4], [4, 4], [5, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 13], [16, 13], [15, 14], [16, 14], [15, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[10, 7], [11, 7], [12, 7], [11, 8], [11, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[6, 10], [6, 11], [7, 11], [7, 12], [8, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "left", "left", "up", "left", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2953/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2953/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2953/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2953/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2953/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2953/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2953/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2953/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 13], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[6, 12], [5, 13], [6, 13], [7, 13], [6, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 4], [2, 5], [3, 5], [3, 6], [4, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 8], [14, 8], [13, 9], [14, 9], [13, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 1], [9, 1], [10, 1], [9, 2], [9, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "left", "left", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards2306/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2306/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2306/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2306/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2306/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 6], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[7, 5], [7, 6], [8, 6], [8, 7], [9, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 11], [12, 11], [13, 11], [12, 12], [12, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 2], [6, 2], [4, 3], [5, 3], [6, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[2, 8], [1, 9], [2, 9], [3, 9], [2, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards2408/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2408/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2408/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2408/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2408/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2408/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2408/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2408/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2408/step_8.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2408/step_9.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2408/step_10.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2408/step_11.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 15], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[14, 14], [13, 15], [14, 15], [15, 15], [14, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 7], [7, 7], [6, 8], [7, 8], [6, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 11], [3, 11], [4, 11], [3, 12], [3, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 3], [8, 4], [9, 4], [9, 5], [10, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "right", "right", "down", "right", "down", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards549/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards549/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards549/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards549/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards549/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards549/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards549/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards549/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards549/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards549/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards549/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards549/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 2], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[5, 1], [6, 1], [5, 2], [6, 2], [5, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[2, 9], [1, 10], [2, 10], [3, 10], [2, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 12], [8, 12], [9, 12], [8, 13], [8, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 4], [13, 4], [11, 5], [12, 5], [13, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "up", "up", "up", "up", "left", "up", "left", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards978/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards978/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards978/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards978/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards978/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards978/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards978/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards978/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards978/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 10], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[1, 9], [2, 9], [3, 9], [2, 10], [2, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 5], [7, 6], [8, 6], [8, 7], [9, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 1], [12, 1], [11, 2], [12, 2], [11, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 14], [15, 14], [13, 15], [14, 15], [15, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["down", "left", "left", "left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards3311/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3311/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3311/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3311/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3311/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3311/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3311/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3311/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 3], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[7, 2], [8, 2], [8, 3], [8, 4], [9, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 12], [14, 13], [15, 13], [15, 14], [16, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 9], [13, 9], [11, 10], [12, 10], [13, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 6], [4, 6], [3, 7], [4, 7], [3, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "up", "left", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards1087/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1087/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1087/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1087/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1087/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1087/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1087/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1087/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 7], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[3, 6], [4, 6], [5, 6], [4, 7], [4, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 14], [10, 15], [11, 15], [12, 15], [11, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 2], [14, 3], [15, 3], [15, 4], [16, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 11], [9, 11], [8, 12], [9, 12], [8, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "up", "up", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1022/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1022/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1022/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1022/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1022/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1022/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1022/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1022/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 9], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[15, 8], [16, 8], [17, 8], [16, 9], [16, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 5], [10, 5], [8, 6], [9, 6], [10, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 11], [12, 12], [13, 12], [13, 13], [14, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [3, 1], [3, 2], [3, 3], [4, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1431/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1431/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1431/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1431/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1431/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1431/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1431/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 11], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[12, 10], [14, 10], [12, 11], [13, 11], [14, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 6], [6, 6], [5, 7], [6, 7], [5, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 3], [16, 3], [16, 4], [16, 5], [17, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[10, 14], [9, 15], [10, 15], [11, 15], [10, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right", "down", "right", "right", "down"]}], [{"image": "training_data/easy/boards/boards1683/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1683/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1683/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1683/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1683/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1683/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1683/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1683/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1683/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 7], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[2, 6], [4, 6], [2, 7], [3, 7], [4, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 1], [9, 1], [10, 1], [9, 2], [9, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 15], [16, 15], [15, 16], [16, 16], [15, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 11], [12, 11], [12, 12], [12, 13], [13, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "up", "left", "left", "left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards1207/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1207/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1207/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1207/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1207/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1207/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1207/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1207/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1207/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1207/step_9.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 4], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[12, 3], [14, 3], [12, 4], [13, 4], [14, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 13], [1, 14], [2, 14], [3, 14], [2, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[7, 9], [8, 9], [7, 10], [8, 10], [7, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 6], [16, 6], [17, 6], [16, 7], [16, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "right", "up", "right", "up", "up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards3094/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3094/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3094/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3094/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3094/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3094/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3094/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3094/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3094/step_8.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 7], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[2, 6], [3, 6], [3, 7], [3, 8], [4, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[6, 3], [5, 4], [6, 4], [7, 4], [6, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[10, 12], [11, 12], [10, 13], [11, 13], [10, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 15], [13, 15], [14, 15], [13, 16], [13, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "left", "left", "left", "left", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards454/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards454/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards454/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards454/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards454/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards454/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards454/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards454/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards454/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards454/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards454/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards454/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards454/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards454/step_13.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 2], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[3, 1], [4, 1], [3, 2], [4, 2], [3, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 9], [12, 10], [13, 10], [13, 11], [14, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 14], [8, 14], [8, 15], [8, 16], [9, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 5], [16, 5], [17, 5], [16, 6], [16, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "left", "up", "left", "left", "up", "left", "left", "up", "up", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards534/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards534/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards534/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards534/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards534/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards534/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards534/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards534/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards534/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards534/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards534/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 12], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[16, 11], [17, 11], [16, 12], [17, 12], [16, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[13, 8], [12, 9], [13, 9], [14, 9], [13, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 1], [9, 1], [9, 2], [9, 3], [10, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[4, 14], [4, 15], [5, 15], [5, 16], [6, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "down", "down", "right", "right", "right", "down", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1152/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1152/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1152/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1152/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1152/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1152/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1152/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1152/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1152/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1152/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1152/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 2], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[5, 1], [6, 1], [7, 1], [6, 2], [6, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[9, 5], [10, 5], [9, 6], [10, 6], [9, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 14], [14, 15], [15, 15], [16, 15], [15, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 11], [13, 11], [11, 12], [12, 12], [13, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "up", "up", "up", "up", "left", "left", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards45/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards45/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards45/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards45/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards45/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards45/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards45/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards45/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards45/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards45/step_9.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards45/step_10.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 12], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[2, 11], [3, 11], [2, 12], [3, 12], [2, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[5, 5], [6, 5], [7, 5], [6, 6], [6, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 15], [14, 15], [12, 16], [13, 16], [14, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 8], [9, 9], [10, 9], [10, 10], [11, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "left", "down", "down", "left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards3339/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3339/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3339/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3339/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 7], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[9, 6], [10, 6], [10, 7], [10, 8], [11, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 15], [15, 15], [16, 15], [15, 16], [15, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[4, 10], [5, 10], [4, 11], [5, 11], [4, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 3], [6, 4], [7, 4], [8, 4], [7, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [7, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "up", "up"]}], [{"image": "training_data/easy/boards/boards68/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards68/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards68/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards68/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards68/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards68/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards68/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards68/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards68/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards68/step_9.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards68/step_10.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards68/step_11.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 15], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[14, 14], [15, 14], [14, 15], [15, 15], [14, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 7], [6, 7], [7, 7], [6, 8], [6, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 11], [2, 12], [3, 12], [4, 12], [3, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 3], [8, 4], [9, 4], [9, 5], [10, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "down", "down", "right", "right", "down", "down", "down", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards385/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards385/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards385/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards385/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards385/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards385/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards385/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards385/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 15], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[10, 14], [11, 14], [10, 15], [11, 15], [10, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 1], [3, 1], [4, 1], [3, 2], [3, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[16, 7], [15, 8], [16, 8], [17, 8], [16, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 11], [5, 12], [6, 12], [6, 13], [7, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "down", "right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards2920/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2920/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2920/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2920/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2920/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 8], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[6, 7], [5, 8], [6, 8], [7, 8], [6, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 2], [11, 2], [9, 3], [10, 3], [11, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 12], [16, 12], [16, 13], [16, 14], [17, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[1, 15], [1, 16], [2, 16], [2, 17], [3, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1617/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1617/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1617/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1617/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1617/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1617/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1617/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1617/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1617/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 14], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[5, 13], [7, 13], [5, 14], [6, 14], [7, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[8, 10], [9, 10], [10, 10], [9, 11], [9, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 3], [16, 3], [16, 4], [16, 5], [17, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[12, 7], [12, 8], [13, 8], [13, 9], [14, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "down", "left", "down", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards1908/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1908/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1908/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1908/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1908/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1908/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1908/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1908/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1908/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 14], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[11, 13], [11, 14], [12, 14], [12, 15], [13, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[9, 10], [10, 10], [9, 11], [10, 11], [9, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 5], [17, 5], [15, 6], [16, 6], [17, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 1], [6, 1], [7, 1], [6, 2], [6, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "down", "down", "down", "down", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards424/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards424/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards424/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards424/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 8], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[11, 7], [12, 7], [11, 8], [12, 8], [11, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 1], [15, 1], [13, 2], [14, 2], [15, 2]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[6, 15], [5, 16], [6, 16], [7, 16], [6, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 10], [1, 11], [2, 11], [2, 12], [3, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "up"]}], [{"image": "training_data/easy/boards/boards287/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards287/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards287/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards287/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards287/step_4.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards287/step_5.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards287/step_6.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards287/step_7.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 5], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[6, 4], [7, 4], [6, 5], [7, 5], [6, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 13], [10, 13], [11, 13], [10, 14], [10, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "up", "up", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards2666/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2666/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2666/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2666/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2666/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2666/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2666/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2666/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 8], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[3, 7], [2, 8], [3, 8], [4, 8], [3, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 15], [14, 15], [12, 16], [13, 16], [14, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 10], [10, 10], [10, 11], [10, 12], [11, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 3], [15, 4], [16, 4], [16, 5], [17, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "left", "left", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards1957/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1957/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1957/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1957/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1957/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1957/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1957/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1957/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 11], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[3, 10], [3, 11], [4, 11], [4, 12], [5, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 2], [12, 2], [11, 3], [12, 3], [11, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 13], [17, 13], [15, 14], [16, 14], [17, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 7], [8, 7], [9, 7], [8, 8], [8, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "down", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards43/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards43/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards43/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 8], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[10, 7], [11, 7], [10, 8], [11, 8], [10, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 14], [4, 14], [5, 14], [4, 15], [4, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[14, 2], [14, 3], [15, 3], [15, 4], [16, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[7, 11], [6, 12], [7, 12], [8, 12], [7, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "up"]}], [{"image": "training_data/easy/boards/boards2446/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2446/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2446/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2446/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2446/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2446/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2446/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2446/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2446/step_8.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 11], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[15, 10], [14, 11], [15, 11], [16, 11], [15, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 13], [3, 13], [2, 14], [3, 14], [2, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 3], [5, 4], [6, 4], [6, 5], [7, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[9, 7], [10, 7], [10, 8], [10, 9], [11, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "down", "right", "right", "right", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards3057/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3057/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3057/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3057/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3057/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3057/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 13], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[9, 12], [10, 12], [10, 13], [10, 14], [11, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[5, 5], [6, 5], [7, 5], [6, 6], [6, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 9], [2, 10], [3, 10], [3, 11], [4, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 2], [14, 2], [13, 3], [14, 3], [13, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards2831/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2831/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2831/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2831/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2831/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2831/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2831/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2831/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2831/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2831/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2831/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2831/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2831/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2831/step_13.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 3], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[2, 2], [1, 3], [2, 3], [3, 3], [2, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 5], [14, 5], [13, 6], [14, 6], [13, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 9], [8, 9], [8, 10], [8, 11], [9, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 15], [5, 15], [6, 15], [5, 16], [5, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["left", "left", "left", "left", "up", "left", "up", "up", "left", "up", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards421/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards421/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards421/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards421/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards421/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards421/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards421/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards421/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards421/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards421/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards421/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards421/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards421/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards421/step_13.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards421/step_14.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 16], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[2, 15], [3, 15], [2, 16], [3, 16], [2, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[6, 12], [5, 13], [6, 13], [7, 13], [6, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 8], [13, 8], [14, 8], [13, 9], [13, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[15, 1], [17, 1], [15, 2], [16, 2], [17, 2]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "left", "down", "down", "down", "left", "down", "down", "down", "down", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2781/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2781/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2781/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2781/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2781/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2781/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2781/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 15], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[9, 14], [8, 15], [9, 15], [10, 15], [9, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 4], [5, 4], [6, 4], [5, 5], [5, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 7], [15, 7], [14, 8], [15, 8], [14, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards3061/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3061/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3061/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 7], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[8, 6], [9, 6], [9, 7], [9, 8], [10, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 11], [14, 12], [15, 12], [16, 12], [15, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 14], [7, 14], [6, 15], [7, 15], [6, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 2], [3, 2], [4, 2], [3, 3], [3, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}], "path": ["up", "up"]}], [{"image": "training_data/easy/boards/boards1730/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1730/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1730/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1730/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1730/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1730/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1730/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1730/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1730/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 2], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[9, 1], [11, 1], [9, 2], [10, 2], [11, 2]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 15], [1, 16], [2, 16], [2, 17], [3, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 12], [6, 12], [6, 13], [6, 14], [7, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 9], [13, 9], [14, 9], [13, 10], [13, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "right", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards3445/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3445/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3445/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3445/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3445/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3445/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3445/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3445/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3445/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3445/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3445/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3445/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3445/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 14], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[15, 13], [16, 13], [16, 14], [16, 15], [17, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 10], [6, 10], [5, 11], [6, 11], [5, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 1], [10, 1], [11, 1], [10, 2], [10, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 6], [12, 7], [13, 7], [14, 7], [13, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "right", "down", "down", "right", "down", "right", "down", "right", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards3099/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3099/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3099/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3099/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3099/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3099/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3099/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3099/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 4], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[10, 3], [11, 3], [11, 4], [11, 5], [12, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 6], [3, 6], [2, 7], [3, 7], [2, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[6, 10], [6, 11], [7, 11], [7, 12], [8, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 13], [15, 14], [16, 14], [17, 14], [16, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "up", "up", "up", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards976/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards976/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards976/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards976/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards976/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards976/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards976/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards976/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards976/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards976/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards976/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 12], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[1, 11], [2, 11], [3, 11], [2, 12], [2, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 15], [13, 15], [11, 16], [12, 16], [13, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[5, 8], [4, 9], [5, 9], [6, 9], [5, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 4], [9, 4], [8, 5], [9, 5], [8, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "down", "left", "left", "left", "left", "down", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards1625/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1625/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1625/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1625/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1625/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1625/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1625/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1625/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1625/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1625/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1625/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1625/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1625/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 14], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[15, 13], [17, 13], [15, 14], [16, 14], [17, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 10], [6, 10], [5, 11], [6, 11], [5, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 1], [10, 1], [11, 1], [10, 2], [10, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 6], [12, 7], [13, 7], [14, 7], [13, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "right", "right", "down", "down", "right", "right", "down", "right", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards2163/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2163/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2163/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2163/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2163/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2163/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2163/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2163/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 6], "target_shape": "W", "target_color": "yellow", "info": [{"piece_grids": [[12, 5], [12, 6], [13, 6], [13, 7], [14, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 13], [7, 13], [7, 14], [7, 15], [8, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 2], [17, 2], [16, 3], [17, 3], [16, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[4, 9], [3, 10], [4, 10], [5, 10], [4, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "up", "right", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards574/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards574/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards574/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards574/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards574/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards574/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 6], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[11, 5], [12, 5], [11, 6], [12, 6], [11, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 9], [6, 9], [4, 10], [5, 10], [6, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[2, 2], [1, 3], [2, 3], [3, 3], [2, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 12], [13, 13], [14, 13], [14, 14], [15, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["right", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards3549/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3549/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3549/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3549/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3549/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 8], "target_shape": "Z", "target_color": "magenta", "info": [{"piece_grids": [[11, 7], [12, 7], [12, 8], [12, 9], [13, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 13], [2, 13], [3, 13], [2, 14], [2, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 4], [16, 4], [15, 5], [16, 5], [15, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 10], [8, 11], [9, 11], [9, 12], [10, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards1061/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1061/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1061/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1061/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1061/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1061/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1061/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 13], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[10, 12], [11, 12], [12, 12], [11, 13], [11, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 1], [2, 2], [3, 2], [3, 3], [4, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 5], [16, 5], [15, 6], [16, 6], [15, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 8], [8, 8], [6, 9], [7, 9], [8, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "down", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards418/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards418/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards418/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards418/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards418/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards418/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards418/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards418/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards418/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards418/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards418/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards418/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 16], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[13, 15], [14, 15], [13, 16], [14, 16], [13, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[15, 4], [15, 5], [16, 5], [16, 6], [17, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[4, 10], [5, 10], [5, 11], [5, 12], [6, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[1, 1], [3, 1], [1, 2], [2, 2], [3, 2]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "down", "down", "down", "down", "right", "right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards361/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards361/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards361/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards361/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards361/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards361/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards361/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards361/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards361/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards361/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards361/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards361/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards361/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards361/step_13.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards361/step_14.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 2], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[2, 1], [3, 1], [2, 2], [3, 2], [2, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 10], [8, 10], [6, 11], [7, 11], [8, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 15], [10, 16], [11, 16], [11, 17], [12, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 6], [16, 6], [17, 6], [16, 7], [16, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "left", "up", "left", "up", "up", "left", "left", "left", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards225/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards225/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 10], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[9, 9], [10, 9], [9, 10], [10, 10], [9, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 4], [6, 4], [4, 5], [5, 5], [6, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[2, 1], [1, 2], [2, 2], [3, 2], [2, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 12], [13, 13], [14, 13], [14, 14], [15, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["down"]}], [{"image": "training_data/easy/boards/boards2573/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2573/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2573/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2573/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2573/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2573/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2573/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2573/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2573/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2573/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 7], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[16, 6], [15, 7], [16, 7], [17, 7], [16, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 2], [9, 2], [8, 3], [9, 3], [8, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 12], [3, 12], [4, 12], [3, 13], [3, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[10, 9], [10, 10], [11, 10], [11, 11], [12, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "right", "right", "right", "right", "right", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards3009/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3009/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3009/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3009/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3009/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3009/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3009/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 11], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[12, 10], [13, 10], [13, 11], [13, 12], [14, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 3], [6, 4], [7, 4], [7, 5], [8, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[3, 15], [4, 15], [3, 16], [4, 16], [3, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[9, 6], [10, 6], [11, 6], [10, 7], [10, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "right", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards752/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards752/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards752/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards752/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards752/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards752/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards752/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards752/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 9], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[15, 8], [16, 8], [17, 8], [16, 9], [16, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [2, 2], [3, 2], [3, 3], [4, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 15], [9, 15], [7, 16], [8, 16], [9, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 4], [12, 5], [13, 5], [14, 5], [13, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2948/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2948/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2948/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2948/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2948/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2948/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2948/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2948/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2948/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 14], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[12, 13], [11, 14], [12, 14], [13, 14], [12, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[9, 10], [10, 10], [9, 11], [10, 11], [9, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 5], [15, 6], [16, 6], [16, 7], [17, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 1], [6, 1], [7, 1], [6, 2], [6, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "down", "down", "right", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards1161/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1161/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1161/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1161/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1161/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1161/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1161/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1161/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1161/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1161/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1161/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 6], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[1, 5], [2, 5], [3, 5], [2, 6], [2, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[16, 12], [17, 12], [16, 13], [17, 13], [16, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 2], [9, 2], [9, 3], [9, 4], [10, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 8], [4, 9], [5, 9], [6, 9], [5, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["left", "left", "left", "left", "up", "up", "left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards2360/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2360/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2360/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2360/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2360/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2360/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2360/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 14], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[7, 13], [7, 14], [8, 14], [8, 15], [9, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 5], [2, 5], [2, 6], [2, 7], [3, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 2], [16, 2], [14, 3], [15, 3], [16, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 9], [4, 10], [5, 10], [6, 10], [5, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "left", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1936/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1936/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1936/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1936/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1936/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1936/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1936/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1936/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 10], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[14, 9], [14, 10], [15, 10], [15, 11], [16, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[3, 5], [4, 5], [4, 6], [4, 7], [5, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 1], [10, 1], [9, 2], [10, 2], [9, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 14], [11, 15], [12, 15], [13, 15], [12, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right", "right", "right", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards1533/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1533/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1533/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1533/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1533/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1533/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1533/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1533/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1533/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1533/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 7], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[15, 6], [17, 6], [15, 7], [16, 7], [17, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 2], [9, 2], [8, 3], [9, 3], [8, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 12], [3, 12], [4, 12], [3, 13], [3, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[10, 9], [10, 10], [11, 10], [11, 11], [12, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "up", "right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1503/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1503/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1503/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1503/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1503/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1503/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1503/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1503/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1503/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1503/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1503/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 5], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[14, 4], [16, 4], [14, 5], [15, 5], [16, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 13], [3, 13], [2, 14], [3, 14], [2, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[9, 7], [10, 7], [10, 8], [10, 9], [11, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 10], [5, 10], [6, 10], [5, 11], [5, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "up", "right", "right", "right", "right", "up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards2914/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2914/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2914/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2914/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2914/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2914/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 6], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[11, 5], [10, 6], [11, 6], [12, 6], [11, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 9], [6, 9], [4, 10], [5, 10], [6, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[1, 2], [2, 2], [3, 2], [2, 3], [2, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 12], [13, 13], [14, 13], [14, 14], [15, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "up", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards3572/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3572/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3572/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3572/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3572/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3572/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "Z", "target_color": "magenta", "info": [{"piece_grids": [[7, 12], [8, 12], [8, 13], [8, 14], [9, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 1], [11, 2], [12, 2], [13, 2], [12, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 15], [3, 15], [1, 16], [2, 16], [3, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[4, 6], [5, 6], [6, 6], [5, 7], [5, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["left", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards2251/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2251/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2251/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2251/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2251/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2251/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2251/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2251/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 10], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[2, 9], [2, 10], [3, 10], [3, 11], [4, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 14], [15, 14], [16, 14], [15, 15], [15, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 3], [9, 3], [8, 4], [9, 4], [8, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 6], [13, 6], [11, 7], [12, 7], [13, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "left", "down", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2694/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2694/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2694/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2694/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2694/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2694/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2694/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2694/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2694/step_8.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2694/step_9.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2694/step_10.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 14], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[14, 13], [13, 14], [14, 14], [15, 14], [14, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[3, 2], [4, 2], [5, 2], [4, 3], [4, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 6], [9, 6], [7, 7], [8, 7], [9, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 9], [11, 9], [11, 10], [11, 11], [12, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "down", "down", "right", "right", "right", "down", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards3437/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3437/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3437/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3437/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3437/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3437/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3437/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3437/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3437/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 14], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[5, 13], [6, 13], [6, 14], [6, 15], [7, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[8, 10], [9, 10], [10, 10], [9, 11], [9, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 3], [17, 3], [15, 4], [16, 4], [17, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[12, 7], [12, 8], [13, 8], [13, 9], [14, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "down", "down", "left", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards2561/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2561/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2561/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 10], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[8, 9], [7, 10], [8, 10], [9, 10], [8, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 6], [11, 6], [11, 7], [11, 8], [12, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 3], [13, 4], [14, 4], [14, 5], [15, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 12], [4, 12], [3, 13], [4, 13], [3, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["left", "down"]}], [{"image": "training_data/easy/boards/boards767/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards767/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards767/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards767/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards767/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards767/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards767/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards767/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards767/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards767/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards767/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards767/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards767/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 2], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[3, 1], [4, 1], [5, 1], [4, 2], [4, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 14], [9, 14], [7, 15], [8, 15], [9, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 9], [15, 10], [16, 10], [16, 11], [17, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[11, 5], [10, 6], [11, 6], [12, 6], [11, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "up", "left", "up", "left", "up", "up", "up", "left", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards517/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards517/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards517/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 8], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[8, 7], [9, 7], [8, 8], [9, 8], [8, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 10], [12, 11], [13, 11], [14, 11], [13, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [4, 1], [2, 2], [3, 2], [4, 2]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 14], [16, 14], [17, 14], [16, 15], [16, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "up"]}], [{"image": "training_data/easy/boards/boards2777/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2777/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2777/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2777/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2777/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2777/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2777/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2777/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2777/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2777/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 12], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[15, 11], [14, 12], [15, 12], [16, 12], [15, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[3, 15], [3, 16], [4, 16], [4, 17], [5, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[12, 6], [13, 6], [12, 7], [13, 7], [12, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[8, 2], [9, 2], [9, 3], [9, 4], [10, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "right", "right", "down", "right", "right", "right", "down"]}], [{"image": "training_data/easy/boards/boards18/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards18/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards18/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards18/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards18/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards18/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards18/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards18/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards18/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 15], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[11, 14], [12, 14], [11, 15], [12, 15], [11, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[5, 4], [6, 4], [6, 5], [6, 6], [7, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 9], [13, 10], [14, 10], [15, 10], [14, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [4, 1], [2, 2], [3, 2], [4, 2]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "down", "down", "down", "right", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards399/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards399/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards399/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards399/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards399/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards399/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards399/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards399/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 6], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[13, 5], [14, 5], [13, 6], [14, 6], [13, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 12], [9, 12], [9, 13], [9, 14], [10, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 8], [16, 8], [17, 8], [16, 9], [16, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 1], [5, 2], [6, 2], [6, 3], [7, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "right", "up", "up", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards530/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards530/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards530/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards530/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards530/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards530/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards530/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 11], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[13, 10], [14, 10], [13, 11], [14, 11], [13, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 15], [10, 15], [8, 16], [9, 16], [10, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 4], [2, 5], [3, 5], [4, 5], [3, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "down", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards3082/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3082/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3082/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3082/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3082/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3082/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3082/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3082/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3082/step_8.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3082/step_9.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3082/step_10.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3082/step_11.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 5], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[1, 4], [2, 4], [2, 5], [2, 6], [3, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 11], [12, 12], [13, 12], [14, 12], [13, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 7], [10, 7], [9, 8], [10, 8], [9, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 1], [7, 1], [5, 2], [6, 2], [7, 2]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "left", "left", "up", "left", "up", "up", "left", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards401/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards401/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards401/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards401/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards401/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 10], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[6, 9], [7, 9], [6, 10], [7, 10], [6, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 14], [9, 15], [10, 15], [10, 16], [11, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 6], [13, 7], [14, 7], [15, 7], [14, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 2], [2, 2], [2, 3], [2, 4], [3, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards494/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards494/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards494/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards494/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards494/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards494/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards494/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards494/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards494/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 7], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[3, 6], [4, 6], [3, 7], [4, 7], [3, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[5, 3], [6, 3], [6, 4], [6, 5], [7, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[9, 12], [10, 12], [11, 12], [10, 13], [10, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 15], [12, 16], [13, 16], [14, 16], [13, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "up", "left", "left", "left", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards2913/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2913/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2913/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2913/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2913/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 12], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[8, 11], [7, 12], [8, 12], [9, 12], [8, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 7], [3, 7], [1, 8], [2, 8], [3, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[5, 1], [6, 1], [5, 2], [6, 2], [5, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 4], [13, 4], [14, 4], [13, 5], [13, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards1753/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1753/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1753/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1753/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1753/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1753/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1753/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 5], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[6, 4], [8, 4], [6, 5], [7, 5], [8, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [7, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 9], [12, 9], [12, 10], [12, 11], [13, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 1], [3, 2], [4, 2], [5, 2], [4, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 12], [15, 12], [16, 12], [15, 13], [15, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "left", "up", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards390/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards390/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 8], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[9, 7], [10, 7], [9, 8], [10, 8], [9, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 2], [16, 2], [17, 2], [16, 3], [16, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[13, 14], [12, 15], [13, 15], [14, 15], [13, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 10], [5, 11], [6, 11], [6, 12], [7, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["up"]}], [{"image": "training_data/easy/boards/boards1937/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1937/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1937/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1937/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1937/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 8], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[11, 7], [11, 8], [12, 8], [12, 9], [13, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 15], [8, 15], [9, 15], [8, 16], [8, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 11], [3, 12], [4, 12], [5, 12], [4, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 3], [17, 3], [15, 4], [16, 4], [17, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards1905/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1905/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1905/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1905/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 10], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[10, 9], [10, 10], [11, 10], [11, 11], [12, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 3], [16, 3], [14, 4], [15, 4], [16, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 15], [7, 16], [8, 16], [9, 16], [8, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 12], [3, 12], [3, 13], [3, 14], [4, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "right", "down"]}], [{"image": "training_data/easy/boards/boards3242/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3242/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3242/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3242/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3242/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3242/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3242/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3242/step_7.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3242/step_8.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3242/step_9.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3242/step_10.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 2], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[11, 1], [12, 1], [12, 2], [12, 3], [13, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 4], [14, 5], [15, 5], [16, 5], [15, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 13], [9, 13], [8, 14], [9, 14], [8, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 7], [3, 7], [1, 8], [2, 8], [3, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "up", "right", "up", "right", "up", "up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards1096/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1096/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1096/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1096/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1096/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1096/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1096/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1096/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 9], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[15, 8], [16, 8], [17, 8], [16, 9], [16, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[4, 5], [5, 5], [4, 6], [5, 6], [4, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[6, 15], [6, 16], [7, 16], [7, 17], [8, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 2], [12, 3], [13, 3], [14, 3], [13, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1044/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1044/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1044/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1044/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1044/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1044/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1044/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1044/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1044/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1044/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1044/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1044/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1044/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1044/step_13.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 2], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[14, 1], [15, 1], [16, 1], [15, 2], [15, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 12], [2, 13], [3, 13], [3, 14], [4, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[6, 4], [7, 4], [6, 5], [7, 5], [6, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[9, 15], [8, 16], [9, 16], [10, 16], [9, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["up", "up", "right", "right", "right", "right", "right", "up", "up", "up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards2598/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2598/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2598/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2598/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2598/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2598/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2598/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2598/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2598/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2598/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 14], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[13, 13], [12, 14], [13, 14], [14, 14], [13, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 7], [7, 7], [7, 8], [7, 9], [8, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 10], [17, 10], [15, 11], [16, 11], [17, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 1], [9, 2], [10, 2], [10, 3], [11, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "right", "right", "right", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards1273/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1273/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1273/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1273/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1273/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1273/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1273/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1273/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1273/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1273/step_9.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 7], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[15, 6], [17, 6], [15, 7], [16, 7], [17, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 2], [9, 2], [8, 3], [9, 3], [8, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 12], [3, 12], [4, 12], [3, 13], [3, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[10, 9], [10, 10], [11, 10], [11, 11], [12, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "right", "right", "right", "right", "right", "up", "up"]}], [{"image": "training_data/easy/boards/boards1144/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1144/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1144/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1144/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1144/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1144/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 5], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[7, 4], [8, 4], [9, 4], [8, 5], [8, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 13], [16, 13], [14, 14], [15, 14], [16, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 8], [12, 8], [11, 9], [12, 9], [11, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 1], [3, 1], [3, 2], [3, 3], [4, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}], "path": ["up", "up", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards1427/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1427/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1427/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1427/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1427/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1427/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1427/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1427/step_7.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1427/step_8.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1427/step_9.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1427/step_10.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1427/step_11.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 13], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[15, 12], [17, 12], [15, 13], [16, 13], [17, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 9], [6, 9], [6, 10], [6, 11], [7, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 3], [12, 4], [13, 4], [14, 4], [13, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 6], [2, 7], [3, 7], [3, 8], [4, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "right", "right", "right", "right", "down", "down", "right", "right", "right", "down"]}], [{"image": "training_data/easy/boards/boards1517/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1517/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1517/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1517/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1517/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1517/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1517/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1517/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1517/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1517/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1517/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1517/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1517/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 14], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[1, 13], [3, 13], [1, 14], [2, 14], [3, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[9, 6], [8, 7], [9, 7], [10, 7], [9, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 10], [15, 10], [16, 10], [15, 11], [15, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[11, 2], [11, 3], [12, 3], [12, 4], [13, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "left", "left", "down", "left", "down", "left", "down", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards2477/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2477/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2477/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2477/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2477/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2477/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2477/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2477/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 11], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[4, 10], [3, 11], [4, 11], [5, 11], [4, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 2], [12, 2], [11, 3], [12, 3], [11, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 13], [15, 14], [16, 14], [16, 15], [17, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 7], [8, 7], [9, 7], [8, 8], [8, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "left", "left", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards1181/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1181/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1181/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1181/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1181/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 10], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[5, 9], [6, 9], [7, 9], [6, 10], [6, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 14], [9, 15], [10, 15], [10, 16], [11, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 6], [13, 7], [14, 7], [15, 7], [14, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 2], [2, 2], [2, 3], [2, 4], [3, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}], "path": ["left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards1157/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1157/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1157/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1157/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1157/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 8], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[11, 7], [12, 7], [13, 7], [12, 8], [12, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[8, 15], [7, 16], [8, 16], [9, 16], [8, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 11], [4, 11], [4, 12], [4, 13], [5, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 3], [15, 4], [16, 4], [16, 5], [17, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards1888/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1888/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1888/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1888/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1888/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1888/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1888/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1888/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1888/step_8.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1888/step_9.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1888/step_10.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1888/step_11.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 15], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[13, 14], [13, 15], [14, 15], [14, 16], [15, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 7], [7, 7], [6, 8], [7, 8], [6, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 11], [3, 11], [4, 11], [3, 12], [3, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 3], [10, 3], [8, 4], [9, 4], [10, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "right", "down", "right", "down", "down", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1245/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1245/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1245/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1245/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1245/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1245/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1245/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1245/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1245/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1245/step_9.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1245/step_10.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1245/step_11.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 4], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[2, 3], [4, 3], [2, 4], [3, 4], [4, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 7], [16, 7], [17, 7], [16, 8], [16, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 14], [7, 15], [8, 15], [9, 15], [8, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 11], [13, 11], [12, 12], [13, 12], [12, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "left", "up", "left", "up", "left", "left", "up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards286/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards286/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards286/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards286/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards286/step_4.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards286/step_5.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards286/step_6.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 7], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[5, 6], [6, 6], [5, 7], [6, 7], [5, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[2, 2], [1, 3], [2, 3], [3, 3], [2, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[8, 12], [9, 12], [10, 12], [9, 13], [9, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 9], [14, 9], [14, 10], [14, 11], [15, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "left", "left", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards183/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards183/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards183/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards183/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards183/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards183/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards183/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards183/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards183/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards183/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards183/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards183/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 3], "target_shape": "P", "target_color": "blue", "info": [{"piece_grids": [[14, 2], [15, 2], [14, 3], [15, 3], [14, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[7, 7], [9, 7], [7, 8], [8, 8], [9, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 11], [5, 11], [5, 12], [5, 13], [6, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[1, 14], [2, 14], [3, 14], [2, 15], [2, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "up", "right", "up", "right", "up", "right", "up", "right", "up", "up"]}], [{"image": "training_data/easy/boards/boards2010/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2010/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2010/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2010/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2010/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2010/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2010/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 8], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[3, 7], [3, 8], [4, 8], [4, 9], [5, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 14], [14, 14], [14, 15], [14, 16], [15, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 10], [10, 10], [9, 11], [10, 11], [9, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 10], [9, 10], [7, 11], [8, 11], [9, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards3370/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3370/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3370/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3370/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3370/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3370/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3370/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3370/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3370/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3370/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3370/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3370/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 13], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[15, 12], [16, 12], [16, 13], [16, 14], [17, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 6], [7, 6], [6, 7], [7, 7], [6, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 15], [11, 16], [12, 16], [13, 16], [12, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 3], [3, 3], [1, 4], [2, 4], [3, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "down", "down", "right", "right", "right", "down", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1977/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1977/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1977/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1977/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1977/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1977/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1977/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1977/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1977/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1977/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1977/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 16], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[5, 15], [5, 16], [6, 16], [6, 17], [7, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 6], [2, 6], [3, 6], [2, 7], [2, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 11], [13, 11], [12, 12], [13, 12], [12, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 1], [15, 2], [16, 2], [17, 2], [16, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "down", "down", "down", "left", "down", "left", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards1851/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1851/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1851/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1851/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1851/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1851/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1851/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1851/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1851/step_8.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1851/step_9.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1851/step_10.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 12], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[15, 11], [15, 12], [16, 12], [16, 13], [17, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 15], [10, 15], [9, 16], [10, 16], [9, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 4], [3, 4], [3, 5], [3, 6], [4, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 8], [12, 9], [13, 9], [14, 9], [13, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "right", "down", "right", "right", "down", "right", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards1690/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1690/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 8], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[8, 7], [10, 7], [8, 8], [9, 8], [10, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 2], [17, 2], [16, 3], [17, 3], [16, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[12, 14], [13, 14], [14, 14], [13, 15], [13, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 10], [5, 11], [6, 11], [6, 12], [7, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["up"]}], [{"image": "training_data/easy/boards/boards595/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards595/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards595/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards595/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards595/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards595/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 7], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[6, 6], [7, 6], [6, 7], [7, 7], [6, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 2], [1, 3], [2, 3], [2, 4], [3, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 10], [17, 10], [15, 11], [16, 11], [17, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[12, 15], [13, 15], [13, 16], [13, 17], [14, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "up", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2257/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2257/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2257/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2257/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2257/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2257/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2257/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2257/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2257/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2257/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 12], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[14, 11], [14, 12], [15, 12], [15, 13], [16, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[3, 15], [5, 15], [3, 16], [4, 16], [5, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[12, 6], [13, 6], [12, 7], [13, 7], [12, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 2], [8, 3], [9, 3], [10, 3], [9, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "down", "right", "right", "right", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards1130/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1130/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1130/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1130/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1130/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1130/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1130/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1130/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1130/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1130/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1130/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1130/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 13], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[1, 12], [2, 12], [3, 12], [2, 13], [2, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[8, 7], [9, 7], [8, 8], [9, 8], [8, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 4], [12, 4], [12, 5], [12, 6], [13, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 1], [14, 2], [15, 2], [15, 3], [16, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "down", "left", "left", "left", "down", "down", "down", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2836/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2836/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2836/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2836/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2836/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2836/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2836/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2836/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2836/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 6], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[14, 5], [13, 6], [14, 6], [15, 6], [14, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 10], [6, 10], [7, 10], [6, 11], [6, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 14], [4, 14], [2, 15], [3, 15], [4, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[10, 2], [11, 2], [10, 3], [11, 3], [10, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["up", "up", "right", "right", "right", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards3079/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3079/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3079/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3079/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 7], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[9, 6], [10, 6], [10, 7], [10, 8], [11, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 15], [15, 15], [16, 15], [15, 16], [15, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[4, 10], [5, 10], [4, 11], [5, 11], [4, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 3], [6, 4], [7, 4], [8, 4], [7, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [7, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "up", "up"]}], [{"image": "training_data/easy/boards/boards2693/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2693/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2693/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2693/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2693/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2693/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2693/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2693/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 13], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[6, 12], [5, 13], [6, 13], [7, 13], [6, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 4], [2, 5], [3, 5], [3, 6], [4, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 8], [14, 8], [13, 9], [14, 9], [13, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 1], [9, 1], [10, 1], [9, 2], [9, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "left", "down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards2805/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2805/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2805/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2805/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2805/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2805/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2805/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2805/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2805/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2805/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2805/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2805/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 4], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[3, 3], [2, 4], [3, 4], [4, 4], [3, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 7], [16, 7], [17, 7], [16, 8], [16, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 14], [8, 14], [8, 15], [8, 16], [9, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 11], [13, 11], [12, 12], [13, 12], [12, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "up", "left", "up", "up", "left", "left", "left", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards392/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards392/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards392/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards392/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards392/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards392/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 10], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[13, 9], [14, 9], [13, 10], [14, 10], [13, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[3, 15], [2, 16], [3, 16], [4, 16], [3, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 1], [16, 1], [17, 1], [16, 2], [16, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[7, 12], [7, 13], [8, 13], [8, 14], [9, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right", "right", "right", "down"]}], [{"image": "training_data/easy/boards/boards1145/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1145/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1145/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1145/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1145/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1145/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1145/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1145/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 14], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[6, 13], [7, 13], [8, 13], [7, 14], [7, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [7, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 1], [10, 2], [11, 2], [11, 3], [12, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 4], [3, 4], [3, 5], [3, 6], [4, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 7], [13, 8], [14, 8], [15, 8], [14, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "down", "left", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards2486/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2486/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2486/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2486/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2486/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2486/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2486/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2486/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2486/step_8.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2486/step_9.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2486/step_10.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2486/step_11.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2486/step_12.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2486/step_13.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 3], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[16, 2], [15, 3], [16, 3], [17, 3], [16, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[12, 7], [12, 8], [13, 8], [13, 9], [14, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 11], [9, 11], [7, 12], [8, 12], [9, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 14], [4, 14], [5, 14], [4, 15], [4, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "right", "up", "right", "right", "right", "up", "right", "right", "up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards3066/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3066/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3066/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3066/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3066/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3066/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3066/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3066/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3066/step_8.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3066/step_9.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3066/step_10.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 6], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[15, 5], [16, 5], [16, 6], [16, 7], [17, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[11, 9], [12, 9], [11, 10], [12, 10], [11, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 15], [3, 15], [1, 16], [2, 16], [3, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 12], [5, 13], [6, 13], [6, 14], [7, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["up", "right", "up", "right", "right", "right", "right", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards2792/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2792/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2792/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2792/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2792/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2792/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[8, 12], [7, 13], [8, 13], [9, 13], [8, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 1], [12, 1], [12, 2], [12, 3], [13, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 15], [3, 15], [1, 16], [2, 16], [3, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[4, 6], [5, 6], [6, 6], [5, 7], [5, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards990/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards990/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards990/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards990/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 11], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[9, 10], [10, 10], [11, 10], [10, 11], [10, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 1], [14, 1], [14, 2], [14, 3], [15, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 13], [1, 14], [2, 14], [2, 15], [3, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 5], [7, 5], [5, 6], [6, 6], [7, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "right", "down"]}], [{"image": "training_data/easy/boards/boards2558/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2558/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2558/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2558/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2558/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2558/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2558/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2558/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2558/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2558/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2558/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2558/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2558/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2558/step_13.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 15], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[2, 14], [1, 15], [2, 15], [3, 15], [2, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 9], [5, 10], [6, 10], [6, 11], [7, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 6], [14, 6], [15, 6], [14, 7], [14, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 3], [11, 3], [9, 4], [10, 4], [11, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "down", "left", "down", "left", "down", "left", "down", "left", "down", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards3305/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3305/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3305/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3305/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3305/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3305/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3305/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3305/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3305/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 16], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[7, 15], [8, 15], [8, 16], [8, 17], [9, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 8], [10, 9], [11, 9], [12, 9], [11, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 11], [4, 11], [5, 11], [4, 12], [4, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 1], [16, 1], [14, 2], [15, 2], [16, 2]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "down", "left", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards3007/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3007/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3007/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3007/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3007/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3007/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3007/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 8], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[13, 7], [14, 7], [14, 8], [14, 9], [15, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 15], [11, 15], [9, 16], [10, 16], [11, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 3], [4, 3], [5, 3], [4, 4], [4, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 11], [6, 12], [7, 12], [8, 12], [7, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2282/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2282/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2282/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 9], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[10, 8], [10, 9], [11, 9], [11, 10], [12, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 4], [2, 4], [2, 5], [2, 6], [3, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 14], [5, 15], [6, 15], [7, 15], [6, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 11], [17, 11], [15, 12], [16, 12], [17, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "right"]}], [{"image": "training_data/easy/boards/boards2873/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2873/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2873/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2873/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2873/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2873/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2873/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 15], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[9, 14], [8, 15], [9, 15], [10, 15], [9, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 1], [5, 1], [4, 2], [5, 2], [4, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 8], [14, 8], [15, 8], [14, 9], [14, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 5], [2, 6], [3, 6], [3, 7], [4, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards2460/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2460/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2460/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2460/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2460/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 11], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[7, 10], [6, 11], [7, 11], [8, 11], [7, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 3], [15, 4], [16, 4], [16, 5], [17, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[13, 15], [14, 15], [13, 16], [14, 16], [13, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 7], [10, 7], [10, 8], [10, 9], [11, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards278/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards278/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 9], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[8, 8], [9, 8], [8, 9], [9, 9], [8, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 14], [4, 14], [2, 15], [3, 15], [4, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[10, 3], [11, 3], [12, 3], [11, 4], [11, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 11], [16, 11], [16, 12], [16, 13], [17, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["left"]}], [{"image": "training_data/easy/boards/boards441/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards441/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards441/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards441/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards441/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards441/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards441/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 15], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[9, 14], [10, 14], [9, 15], [10, 15], [9, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[5, 4], [4, 5], [5, 5], [6, 5], [5, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 7], [14, 7], [15, 7], [14, 8], [14, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1926/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1926/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1926/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1926/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1926/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1926/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1926/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1926/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1926/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 11], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[14, 10], [14, 11], [15, 11], [15, 12], [16, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 13], [3, 13], [2, 14], [3, 14], [2, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 3], [7, 3], [5, 4], [6, 4], [7, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[10, 7], [9, 8], [10, 8], [11, 8], [10, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "down", "right", "right", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards2778/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2778/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2778/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2778/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2778/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2778/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2778/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2778/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2778/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2778/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2778/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2778/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2778/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 2], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[14, 1], [13, 2], [14, 2], [15, 2], [14, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 4], [1, 5], [2, 5], [2, 6], [3, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[10, 12], [11, 12], [12, 12], [11, 13], [11, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[7, 8], [9, 8], [7, 9], [8, 9], [9, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "up", "right", "right", "up", "right", "up", "up", "up", "right", "up", "up"]}], [{"image": "training_data/easy/boards/boards684/step_0.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards684/step_1.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards684/step_2.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards684/step_3.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 8], "target_shape": "T", "target_color": "red", "info": [{"piece_grids": [[10, 7], [11, 7], [12, 7], [11, 8], [11, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 1], [15, 1], [13, 2], [14, 2], [15, 2]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[6, 15], [5, 16], [6, 16], [7, 16], [6, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 10], [1, 11], [2, 11], [2, 12], [3, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "right", "right"]}], [{"image": "training_data/easy/boards/boards3036/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3036/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3036/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3036/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3036/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3036/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3036/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3036/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3036/step_8.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3036/step_9.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3036/step_10.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3036/step_11.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3036/step_12.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3036/step_13.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3036/step_14.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 16], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[15, 15], [16, 15], [16, 16], [16, 17], [17, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 4], [11, 5], [12, 5], [12, 6], [13, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[8, 8], [10, 8], [8, 9], [9, 9], [10, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 1], [3, 2], [4, 2], [5, 2], [4, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "down", "down", "down", "down", "right", "right", "down", "right", "right", "right", "right", "down", "down"]}], [{"image": "training_data/easy/boards/boards220/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards220/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards220/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards220/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 8], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[11, 7], [12, 7], [11, 8], [12, 8], [11, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 14], [14, 15], [15, 15], [15, 16], [16, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[2, 2], [4, 2], [2, 3], [3, 3], [4, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 10], [7, 10], [8, 10], [7, 11], [7, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "up", "right"]}], [{"image": "training_data/easy/boards/boards221/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards221/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards221/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 10], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[8, 9], [9, 9], [8, 10], [9, 10], [8, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 6], [11, 6], [11, 7], [11, 8], [12, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 3], [13, 4], [14, 4], [14, 5], [15, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 12], [3, 12], [4, 12], [3, 13], [3, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["down", "left"]}], [{"image": "training_data/easy/boards/boards3465/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3465/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3465/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3465/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 10], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[10, 9], [11, 9], [11, 10], [11, 11], [12, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 3], [14, 4], [15, 4], [15, 5], [16, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 15], [7, 16], [8, 16], [9, 16], [8, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 12], [4, 12], [2, 13], [3, 13], [4, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "down", "right"]}], [{"image": "training_data/easy/boards/boards1367/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1367/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1367/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1367/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1367/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1367/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1367/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1367/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 14], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[6, 13], [8, 13], [6, 14], [7, 14], [8, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 9], [3, 9], [4, 9], [3, 10], [3, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 1], [15, 1], [14, 2], [15, 2], [14, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[10, 4], [11, 4], [11, 5], [11, 6], [12, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "left", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards719/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards719/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards719/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards719/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards719/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards719/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards719/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards719/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards719/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards719/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards719/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards719/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards719/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 14], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[1, 13], [2, 13], [3, 13], [2, 14], [2, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 5], [7, 5], [5, 6], [6, 6], [7, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 8], [12, 8], [11, 9], [12, 9], [11, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "down", "left", "left", "down", "left", "left", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards1336/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1336/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1336/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1336/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1336/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1336/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1336/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1336/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1336/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1336/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1336/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1336/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1336/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1336/step_13.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 2], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[14, 1], [16, 1], [14, 2], [15, 2], [16, 2]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 10], [10, 10], [11, 10], [10, 11], [10, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 4], [2, 5], [3, 5], [3, 6], [4, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 7], [7, 7], [6, 8], [7, 8], [6, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "right", "up", "right", "up", "up", "right", "up", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards1854/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1854/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1854/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1854/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1854/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1854/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1854/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1854/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 3], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[7, 2], [7, 3], [8, 3], [8, 4], [9, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 12], [13, 12], [13, 13], [13, 14], [14, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[2, 15], [3, 15], [2, 16], [3, 16], [2, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[16, 5], [15, 6], [16, 6], [17, 6], [16, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards859/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards859/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards859/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards859/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards859/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards859/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards859/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards859/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 15], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[7, 14], [8, 14], [9, 14], [8, 15], [8, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 1], [3, 1], [1, 2], [2, 2], [3, 2]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[10, 9], [11, 9], [11, 10], [11, 11], [12, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 5], [6, 5], [5, 6], [6, 6], [5, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "left", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1745/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1745/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1745/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1745/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1745/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1745/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1745/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1745/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1745/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 16], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[7, 15], [9, 15], [7, 16], [8, 16], [9, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 8], [10, 9], [11, 9], [12, 9], [11, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 11], [4, 11], [5, 11], [4, 12], [4, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 1], [15, 1], [15, 2], [15, 3], [16, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "down", "down", "down", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards972/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards972/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards972/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards972/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards972/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards972/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[7, 12], [8, 12], [9, 12], [8, 13], [8, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 1], [12, 1], [12, 2], [12, 3], [13, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 15], [3, 15], [1, 16], [2, 16], [3, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 6], [4, 7], [5, 7], [6, 7], [5, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards403/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards403/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards403/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards403/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards403/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards403/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards403/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 13], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[11, 12], [12, 12], [11, 13], [12, 13], [11, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 7], [13, 8], [14, 8], [14, 9], [15, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[4, 15], [5, 15], [6, 15], [5, 16], [5, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[7, 4], [8, 4], [8, 5], [8, 6], [9, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards3507/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3507/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3507/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3507/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3507/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3507/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3507/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3507/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3507/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3507/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3507/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3507/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 13], "target_shape": "Z", "target_color": "magenta", "info": [{"piece_grids": [[15, 12], [16, 12], [16, 13], [16, 14], [17, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 9], [7, 9], [5, 10], [6, 10], [7, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 3], [12, 4], [13, 4], [14, 4], [13, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 6], [2, 7], [3, 7], [3, 8], [4, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "down", "right", "down", "right", "right", "right", "right", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards3584/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3584/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3584/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3584/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 8], "target_shape": "Z", "target_color": "magenta", "info": [{"piece_grids": [[10, 7], [11, 7], [11, 8], [11, 9], [12, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 15], [8, 15], [7, 16], [8, 16], [7, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 2], [14, 3], [15, 3], [16, 3], [15, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[2, 11], [3, 11], [4, 11], [3, 12], [3, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "up"]}], [{"image": "training_data/easy/boards/boards2681/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2681/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2681/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2681/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2681/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2681/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2681/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 7], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[13, 6], [12, 7], [13, 7], [14, 7], [13, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 9], [10, 9], [11, 9], [10, 10], [10, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 1], [5, 1], [5, 2], [5, 3], [6, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 13], [17, 13], [15, 14], [16, 14], [17, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["right", "right", "up", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards707/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards707/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards707/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards707/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards707/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards707/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards707/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards707/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 12], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[4, 11], [5, 11], [6, 11], [5, 12], [5, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[14, 14], [14, 15], [15, 15], [15, 16], [16, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[1, 6], [2, 6], [2, 7], [2, 8], [3, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 3], [12, 3], [10, 4], [11, 4], [12, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "down", "down", "left", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards1104/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1104/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1104/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1104/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1104/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1104/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 8], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[12, 7], [13, 7], [14, 7], [13, 8], [13, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 4], [5, 5], [6, 5], [6, 6], [7, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[9, 15], [10, 15], [10, 16], [10, 17], [11, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 11], [4, 11], [3, 12], [4, 12], [3, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards178/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards178/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards178/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards178/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards178/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards178/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards178/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards178/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards178/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards178/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards178/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards178/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards178/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 2], "target_shape": "P", "target_color": "blue", "info": [{"piece_grids": [[14, 1], [15, 1], [14, 2], [15, 2], [14, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 4], [1, 5], [2, 5], [2, 6], [3, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[11, 12], [10, 13], [11, 13], [12, 13], [11, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[7, 8], [9, 8], [7, 9], [8, 9], [9, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "up", "up", "right", "up", "up", "up", "right", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards2705/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2705/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2705/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2705/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2705/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2705/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2705/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2705/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 14], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[7, 13], [6, 14], [7, 14], [8, 14], [7, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 1], [10, 2], [11, 2], [11, 3], [12, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 4], [3, 4], [3, 5], [3, 6], [4, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 7], [14, 7], [15, 7], [14, 8], [14, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "left", "down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards3261/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3261/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3261/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3261/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3261/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 10], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[5, 9], [6, 9], [6, 10], [6, 11], [7, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 14], [9, 15], [10, 15], [10, 16], [11, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 6], [14, 6], [15, 6], [14, 7], [14, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 2], [1, 3], [2, 3], [3, 3], [2, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["left", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards3196/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3196/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3196/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3196/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3196/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3196/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3196/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3196/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 10], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[2, 9], [3, 9], [3, 10], [3, 11], [4, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 3], [15, 4], [16, 4], [16, 5], [17, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[6, 14], [8, 14], [6, 15], [7, 15], [8, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 6], [11, 6], [12, 6], [11, 7], [11, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1040/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1040/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1040/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1040/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1040/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1040/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1040/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1040/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1040/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 15], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[6, 14], [7, 14], [8, 14], [7, 15], [7, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 10], [3, 11], [4, 11], [4, 12], [5, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 1], [10, 1], [10, 2], [10, 3], [11, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 5], [14, 5], [13, 6], [14, 6], [13, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "left", "down", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards2854/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2854/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2854/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2854/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2854/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2854/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2854/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2854/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2854/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2854/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2854/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 15], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[13, 14], [12, 15], [13, 15], [14, 15], [13, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[1, 8], [2, 8], [3, 8], [2, 9], [2, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 2], [11, 2], [9, 3], [10, 3], [11, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[7, 11], [8, 11], [7, 12], [8, 12], [7, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "right", "down", "down", "down", "right", "right", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards1567/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1567/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1567/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1567/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1567/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1567/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1567/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1567/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1567/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 6], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[13, 5], [15, 5], [13, 6], [14, 6], [15, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 10], [7, 10], [8, 10], [7, 11], [7, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 15], [11, 15], [11, 16], [11, 17], [12, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 1], [3, 2], [4, 2], [5, 2], [4, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "up", "up", "right", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards219/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards219/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards219/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards219/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 7], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[10, 6], [11, 6], [10, 7], [11, 7], [10, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 15], [14, 16], [15, 16], [16, 16], [15, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[3, 10], [4, 10], [5, 10], [4, 11], [4, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[6, 3], [7, 3], [7, 4], [7, 5], [8, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [7, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["up", "right", "up"]}], [{"image": "training_data/easy/boards/boards400/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards400/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards400/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 10], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[10, 9], [11, 9], [10, 10], [11, 10], [10, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 12], [3, 13], [4, 13], [4, 14], [5, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[15, 5], [16, 5], [17, 5], [16, 6], [16, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 1], [7, 1], [7, 2], [7, 3], [8, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [7, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right"]}], [{"image": "training_data/easy/boards/boards2318/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2318/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2318/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2318/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2318/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2318/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2318/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2318/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 4], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[10, 3], [10, 4], [11, 4], [11, 5], [12, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 8], [16, 8], [16, 9], [16, 10], [17, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 11], [7, 11], [8, 11], [7, 12], [7, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 14], [5, 14], [4, 15], [5, 15], [4, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "up", "right", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards1562/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1562/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1562/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1562/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1562/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1562/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1562/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 14], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[9, 13], [11, 13], [9, 14], [10, 14], [11, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 10], [13, 10], [13, 11], [13, 12], [14, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[16, 6], [17, 6], [16, 7], [17, 7], [16, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 2], [7, 2], [8, 2], [7, 3], [7, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards3421/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3421/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3421/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3421/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3421/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3421/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3421/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3421/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3421/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3421/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 16], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[6, 15], [7, 15], [7, 16], [7, 17], [8, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 6], [4, 6], [3, 7], [4, 7], [3, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 12], [15, 12], [13, 13], [14, 13], [15, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 1], [10, 1], [11, 1], [10, 2], [10, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "left", "down", "down", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards167/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards167/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards167/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards167/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards167/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards167/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards167/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards167/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards167/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards167/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 4], "target_shape": "P", "target_color": "blue", "info": [{"piece_grids": [[13, 3], [14, 3], [13, 4], [14, 4], [13, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 13], [2, 13], [2, 14], [2, 15], [3, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[6, 9], [7, 9], [8, 9], [7, 10], [7, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 6], [15, 7], [16, 7], [17, 7], [16, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "right", "up", "up", "right", "up", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards11/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards11/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards11/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards11/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards11/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards11/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards11/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards11/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards11/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 8], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[16, 7], [17, 7], [16, 8], [17, 8], [16, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 4], [11, 4], [9, 5], [10, 5], [11, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[6, 13], [7, 13], [7, 14], [7, 15], [8, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 10], [12, 11], [13, 11], [14, 11], [13, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "up", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1001/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1001/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1001/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 10], "target_shape": "T", "target_color": "cyan", "info": [{"piece_grids": [[7, 9], [8, 9], [9, 9], [8, 10], [8, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 6], [11, 6], [11, 7], [11, 8], [12, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 3], [13, 4], [14, 4], [14, 5], [15, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 12], [4, 12], [3, 13], [4, 13], [3, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["left", "down"]}], [{"image": "training_data/easy/boards/boards571/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards571/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards571/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards571/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards571/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 10], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[12, 9], [13, 9], [12, 10], [13, 10], [12, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 6], [5, 6], [6, 6], [5, 7], [5, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 1], [8, 1], [8, 2], [8, 3], [9, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 14], [3, 14], [1, 15], [2, 15], [3, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["down", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards703/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards703/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards703/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards703/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards703/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards703/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards703/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards703/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards703/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards703/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards703/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards703/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 3], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[13, 2], [14, 2], [15, 2], [14, 3], [14, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[7, 7], [9, 7], [7, 8], [8, 8], [9, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 11], [5, 11], [5, 12], [5, 13], [6, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[2, 14], [3, 14], [2, 15], [3, 15], [2, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "up", "right", "up", "right", "right", "up", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2623/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2623/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2623/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2623/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2623/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2623/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2623/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2623/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2623/step_8.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 16], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[10, 15], [9, 16], [10, 16], [11, 16], [10, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 10], [5, 10], [5, 11], [5, 12], [6, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[1, 3], [2, 3], [3, 3], [2, 4], [2, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 6], [16, 6], [15, 7], [16, 7], [15, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "down", "down", "right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards395/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards395/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards395/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards395/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards395/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards395/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards395/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards395/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 3], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[10, 2], [11, 2], [10, 3], [11, 3], [10, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 10], [2, 10], [2, 11], [2, 12], [3, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 6], [14, 6], [15, 6], [14, 7], [14, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 13], [4, 14], [5, 14], [6, 14], [5, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up", "up", "up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards2631/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2631/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2631/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2631/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2631/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2631/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2631/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2631/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2631/step_8.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2631/step_9.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2631/step_10.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 12], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[16, 11], [15, 12], [16, 12], [17, 12], [16, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[9, 15], [10, 15], [9, 16], [10, 16], [9, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 4], [4, 4], [2, 5], [3, 5], [4, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 8], [13, 8], [13, 9], [13, 10], [14, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "down", "right", "right", "right", "right", "right", "right", "down", "down"]}], [{"image": "training_data/easy/boards/boards2149/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2149/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2149/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2149/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2149/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2149/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 11], "target_shape": "W", "target_color": "yellow", "info": [{"piece_grids": [[11, 10], [11, 11], [12, 11], [12, 12], [13, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 1], [2, 2], [3, 2], [4, 2], [3, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 7], [15, 7], [15, 8], [15, 9], [16, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 4], [8, 4], [9, 4], [8, 5], [8, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "down", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards852/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards852/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards852/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards852/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards852/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards852/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 5], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[7, 4], [8, 4], [9, 4], [8, 5], [8, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[10, 15], [11, 15], [11, 16], [11, 17], [12, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 7], [14, 8], [15, 8], [15, 9], [16, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 11], [1, 12], [2, 12], [3, 12], [2, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["left", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards855/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards855/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards855/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards855/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards855/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards855/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 7], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[5, 6], [6, 6], [7, 6], [6, 7], [6, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[1, 2], [1, 3], [2, 3], [2, 4], [3, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 10], [17, 10], [15, 11], [16, 11], [17, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[12, 15], [13, 15], [13, 16], [13, 17], [14, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "left", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards2786/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2786/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2786/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2786/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 11], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[10, 10], [9, 11], [10, 11], [11, 11], [10, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 14], [4, 14], [3, 15], [4, 15], [3, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 7], [7, 7], [5, 8], [6, 8], [7, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 3], [14, 3], [14, 4], [14, 5], [15, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "down", "down"]}], [{"image": "training_data/easy/boards/boards2539/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2539/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2539/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2539/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2539/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2539/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2539/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2539/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2539/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2539/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2539/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2539/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2539/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 14], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[2, 13], [1, 14], [2, 14], [3, 14], [2, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 5], [7, 5], [5, 6], [6, 6], [7, 6]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 8], [12, 8], [11, 9], [12, 9], [11, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "down", "down", "left", "left", "left", "left", "down", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards3248/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3248/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3248/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3248/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3248/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3248/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3248/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3248/step_7.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3248/step_8.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3248/step_9.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3248/step_10.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 15], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[4, 14], [5, 14], [5, 15], [5, 16], [6, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[13, 9], [13, 10], [14, 10], [14, 11], [15, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 3], [8, 3], [9, 3], [8, 4], [8, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 6], [12, 6], [11, 7], [12, 7], [11, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "down", "down", "down", "down", "left", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards3525/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3525/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3525/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3525/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3525/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3525/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3525/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 11], "target_shape": "Z", "target_color": "magenta", "info": [{"piece_grids": [[12, 10], [13, 10], [13, 11], [13, 12], [14, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 14], [7, 14], [6, 15], [7, 15], [6, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[8, 7], [8, 8], [9, 8], [9, 9], [10, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 1], [3, 1], [4, 1], [3, 2], [3, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "down", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards1757/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1757/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1757/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1757/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1757/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1757/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 13], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[9, 12], [11, 12], [9, 13], [10, 13], [11, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[5, 5], [6, 5], [7, 5], [6, 6], [6, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 9], [2, 10], [3, 10], [3, 11], [4, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 2], [14, 2], [13, 3], [14, 3], [13, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards3344/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3344/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3344/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3344/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3344/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3344/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3344/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3344/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 8], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[14, 7], [15, 7], [15, 8], [15, 9], [16, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [15, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[12, 11], [13, 11], [12, 12], [13, 12], [12, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[8, 3], [10, 3], [8, 4], [9, 4], [10, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 4], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 15], [4, 16], [5, 16], [6, 16], [5, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "right", "right", "up", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1815/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1815/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1815/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1815/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1815/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1815/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1815/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1815/step_7.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 8], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[2, 7], [2, 8], [3, 8], [3, 9], [4, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 3], [8, 4], [9, 4], [10, 4], [9, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 12], [14, 12], [15, 12], [14, 13], [14, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 15], [7, 15], [5, 16], [6, 16], [7, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["left", "left", "left", "left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards314/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards314/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards314/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards314/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards314/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards314/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 6], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[11, 5], [12, 5], [11, 6], [12, 6], [11, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 9], [6, 9], [4, 10], [5, 10], [6, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[2, 2], [1, 3], [2, 3], [3, 3], [2, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 12], [13, 13], [14, 13], [14, 14], [15, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["right", "up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards3018/step_0.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3018/step_1.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3018/step_2.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3018/step_3.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3018/step_4.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3018/step_5.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3018/step_6.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3018/step_7.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3018/step_8.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3018/step_9.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3018/step_10.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3018/step_11.png", "prompt": "You are at the black dot in the board. The target is the red Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 16], "target_shape": "Z", "target_color": "red", "info": [{"piece_grids": [[12, 15], [13, 15], [13, 16], [13, 17], [14, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[15, 4], [15, 5], [16, 5], [16, 6], [17, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 10], [4, 11], [5, 11], [6, 11], [5, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[1, 1], [3, 1], [1, 2], [2, 2], [3, 2]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "down", "right", "down", "down", "right", "right", "right", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1686/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1686/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1686/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1686/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1686/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1686/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1686/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 12], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[11, 11], [13, 11], [11, 12], [12, 12], [13, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 4], [10, 4], [9, 5], [10, 5], [9, 6]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 14], [6, 14], [7, 14], [6, 15], [6, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 8], [1, 9], [2, 9], [3, 9], [2, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "down", "right", "right", "down", "down"]}], [{"image": "training_data/easy/boards/boards997/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards997/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards997/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards997/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards997/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards997/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards997/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards997/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards997/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards997/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards997/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards997/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards997/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 14], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[1, 13], [2, 13], [3, 13], [2, 14], [2, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[8, 6], [9, 6], [9, 7], [9, 8], [10, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 10], [14, 11], [15, 11], [16, 11], [15, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[11, 2], [11, 3], [12, 3], [12, 4], [13, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "left", "down", "down", "left", "left", "left", "down", "down", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards817/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards817/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards817/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards817/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards817/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards817/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards817/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards817/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards817/step_8.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards817/step_9.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 3], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[5, 2], [6, 2], [7, 2], [6, 3], [6, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 14], [12, 14], [12, 15], [12, 16], [13, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 9], [4, 9], [2, 10], [3, 10], [4, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 5], [15, 6], [16, 6], [16, 7], [17, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "left", "left", "up", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards1236/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1236/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1236/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1236/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1236/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1236/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1236/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1236/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1236/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1236/step_9.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1236/step_10.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 12], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[1, 11], [3, 11], [1, 12], [2, 12], [3, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[11, 15], [12, 15], [12, 16], [12, 17], [13, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 8], [5, 8], [6, 8], [5, 9], [5, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 4], [9, 4], [8, 5], [9, 5], [8, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "left", "left", "down", "left", "left", "left", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards1134/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1134/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1134/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1134/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1134/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1134/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1134/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1134/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1134/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1134/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1134/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 14], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[13, 13], [14, 13], [15, 13], [14, 14], [14, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 14], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[4, 2], [3, 3], [4, 3], [5, 3], [4, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 6], [9, 6], [7, 7], [8, 7], [9, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 9], [11, 9], [11, 10], [11, 11], [12, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "right", "down", "down", "right", "down", "down", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards1444/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1444/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1444/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1444/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1444/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1444/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1444/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1444/step_7.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1444/step_8.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1444/step_9.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1444/step_10.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 5], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[14, 4], [16, 4], [14, 5], [15, 5], [16, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[4, 8], [3, 9], [4, 9], [5, 9], [4, 10]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 11], [8, 11], [7, 12], [8, 12], [7, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 15], [9, 16], [10, 16], [10, 17], [11, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right", "right", "up", "up", "right", "right", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards1864/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1864/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1864/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1864/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1864/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1864/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1864/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 3], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[8, 2], [8, 3], [9, 3], [9, 4], [10, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 14], [12, 14], [12, 15], [12, 16], [13, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 9], [5, 9], [3, 10], [4, 10], [5, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[16, 6], [15, 7], [16, 7], [17, 7], [16, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards860/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards860/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards860/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards860/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards860/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards860/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 10], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[4, 9], [5, 9], [6, 9], [5, 10], [5, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 3], [11, 3], [10, 4], [11, 4], [10, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 13], [12, 14], [13, 14], [14, 14], [13, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[15, 6], [17, 6], [15, 7], [16, 7], [17, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards913/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards913/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards913/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards913/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards913/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards913/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards913/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards913/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards913/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards913/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards913/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards913/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards913/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards913/step_13.png", "prompt": "You are at the black dot in the board. The target is the yellow T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 3], "target_shape": "T", "target_color": "yellow", "info": [{"piece_grids": [[1, 2], [2, 2], [3, 2], [2, 3], [2, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 13], [8, 13], [8, 14], [8, 15], [9, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 8], [15, 8], [13, 9], [14, 9], [15, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 5], [4, 6], [5, 6], [6, 6], [5, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left", "up", "left", "up", "left", "up", "up", "left", "left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards803/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards803/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards803/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards803/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards803/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards803/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards803/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards803/step_7.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards803/step_8.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 16], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[9, 15], [10, 15], [11, 15], [10, 16], [10, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 10], [5, 10], [5, 11], [5, 12], [6, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[2, 3], [1, 4], [2, 4], [3, 4], [2, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 6], [16, 6], [15, 7], [16, 7], [15, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "right", "down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1846/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1846/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1846/step_2.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1846/step_3.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1846/step_4.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1846/step_5.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1846/step_6.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 7], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[4, 6], [4, 7], [5, 7], [5, 8], [6, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[1, 2], [2, 2], [3, 2], [2, 3], [2, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[9, 12], [10, 12], [9, 13], [10, 13], [9, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 9], [13, 10], [14, 10], [15, 10], [14, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "up", "left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards2629/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2629/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2629/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2629/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2629/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2629/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2629/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2629/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2629/step_8.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2629/step_9.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2629/step_10.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2629/step_11.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 2], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[5, 1], [4, 2], [5, 2], [6, 2], [5, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[1, 9], [2, 9], [3, 9], [2, 10], [2, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 12], [9, 12], [8, 13], [9, 13], [8, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 13], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 4], [13, 4], [11, 5], [12, 5], [13, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}], "path": ["up", "left", "up", "left", "up", "up", "up", "left", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards2592/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2592/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2592/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2592/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2592/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2592/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2592/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2592/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2592/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 6], "target_shape": "X", "target_color": "blue", "info": [{"piece_grids": [[14, 5], [13, 6], [14, 6], [15, 6], [14, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 11], [5, 12], [6, 12], [6, 13], [7, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 8], [3, 8], [3, 9], [3, 10], [4, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 2], [11, 2], [10, 3], [11, 3], [10, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "up", "up", "right", "right", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards824/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards824/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards824/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards824/step_3.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards824/step_4.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards824/step_5.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards824/step_6.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 3], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[8, 2], [9, 2], [10, 2], [9, 3], [9, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 14], [13, 14], [11, 15], [12, 15], [13, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 9], [3, 10], [4, 10], [4, 11], [5, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 6], [16, 6], [16, 7], [16, 8], [17, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards1317/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1317/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1317/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1317/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1317/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1317/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1317/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 9], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[14, 8], [16, 8], [14, 9], [15, 9], [16, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [15, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 5], [6, 6], [7, 6], [7, 7], [8, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 12], [10, 12], [10, 13], [10, 14], [11, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[4, 15], [5, 15], [4, 16], [5, 16], [4, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2110/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2110/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2110/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2110/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2110/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2110/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2110/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2110/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2110/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2110/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2110/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 14], "target_shape": "W", "target_color": "yellow", "info": [{"piece_grids": [[13, 13], [13, 14], [14, 14], [14, 15], [15, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 6], [8, 6], [8, 7], [8, 8], [9, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 3], [11, 3], [12, 3], [11, 4], [11, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 10], [6, 10], [5, 11], [6, 11], [5, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "right", "down", "right", "right", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards8/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards8/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards8/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards8/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards8/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards8/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards8/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards8/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards8/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards8/step_9.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards8/step_10.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards8/step_11.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards8/step_12.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards8/step_13.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 2], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[15, 1], [16, 1], [15, 2], [16, 2], [15, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 12], [6, 12], [6, 13], [6, 14], [7, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 6], [11, 7], [12, 7], [12, 8], [13, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 15], [1, 16], [2, 16], [3, 16], [2, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 16], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "right", "right", "up", "up", "right", "up", "up", "up", "right", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards1462/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1462/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1462/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1462/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1462/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1462/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1462/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1462/step_7.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 7], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[13, 6], [15, 6], [13, 7], [14, 7], [15, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[10, 9], [10, 10], [11, 10], [11, 11], [12, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 2], [8, 2], [8, 3], [8, 4], [9, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 13], [6, 13], [5, 14], [6, 14], [5, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "right", "up", "up", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards3469/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3469/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3469/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3469/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3469/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3469/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3469/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3469/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3469/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3469/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3469/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3469/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3469/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3469/step_13.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 3], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[1, 2], [2, 2], [2, 3], [2, 4], [3, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 8], [12, 9], [13, 9], [13, 10], [14, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 11], [9, 11], [7, 12], [8, 12], [9, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 5], [16, 5], [17, 5], [16, 6], [16, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "left", "up", "left", "left", "up", "up", "left", "left", "up", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards226/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards226/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards226/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards226/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards226/step_4.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 6], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[8, 5], [9, 5], [8, 6], [9, 6], [8, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 11], [11, 12], [12, 12], [13, 12], [12, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 2], [4, 3], [5, 3], [5, 4], [6, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[1, 8], [2, 8], [2, 9], [2, 10], [3, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards2869/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2869/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2869/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2869/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2869/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2869/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2869/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 13], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[7, 12], [6, 13], [7, 13], [8, 13], [7, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[9, 2], [10, 2], [10, 3], [10, 4], [11, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 5], [1, 6], [2, 6], [2, 7], [3, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 15], [15, 15], [13, 16], [14, 16], [15, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "left", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards537/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards537/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards537/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards537/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards537/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards537/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards537/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 9], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[15, 8], [16, 8], [15, 9], [16, 9], [15, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 5], [6, 6], [7, 6], [7, 7], [8, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 12], [11, 12], [9, 13], [10, 13], [11, 13]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 15], [4, 15], [5, 15], [4, 16], [4, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2333/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2333/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2333/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 11], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[8, 10], [8, 11], [9, 11], [9, 12], [10, 12]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 13], [16, 13], [16, 14], [16, 15], [17, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[12, 6], [13, 6], [14, 6], [13, 7], [13, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 2], [5, 3], [6, 3], [7, 3], [6, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down"]}], [{"image": "training_data/easy/boards/boards535/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards535/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards535/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards535/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards535/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards535/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards535/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards535/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards535/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards535/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 4], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[5, 3], [6, 3], [5, 4], [6, 4], [5, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [5, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[13, 7], [15, 7], [13, 8], [14, 8], [15, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 13], [7, 14], [8, 14], [8, 15], [9, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 10], [10, 11], [11, 11], [12, 11], [11, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "up", "left", "left", "left", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2720/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2720/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2720/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2720/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2720/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 11], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[7, 10], [6, 11], [7, 11], [8, 11], [7, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 3], [15, 4], [16, 4], [16, 5], [17, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[13, 15], [14, 15], [13, 16], [14, 16], [13, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 7], [10, 7], [10, 8], [10, 9], [11, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards232/step_0.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards232/step_1.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards232/step_2.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards232/step_3.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards232/step_4.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards232/step_5.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards232/step_6.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards232/step_7.png", "prompt": "You are at the black dot in the board. The target is the green P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 9], "target_shape": "P", "target_color": "green", "info": [{"piece_grids": [[16, 8], [17, 8], [16, 9], [17, 9], [16, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [2, 2], [3, 2], [3, 3], [4, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 15], [9, 15], [7, 16], [8, 16], [9, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 4], [12, 5], [13, 5], [14, 5], [13, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2279/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2279/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2279/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2279/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2279/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2279/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2279/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2279/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2279/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2279/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2279/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2279/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2279/step_12.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 14], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[1, 13], [1, 14], [2, 14], [2, 15], [3, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 5], [6, 5], [6, 6], [6, 7], [7, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 1], [15, 2], [16, 2], [17, 2], [16, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 8], [12, 8], [11, 9], [12, 9], [11, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "left", "left", "down", "down", "down", "down", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1108/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1108/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1108/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1108/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1108/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1108/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1108/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1108/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1108/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1108/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1108/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1108/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 15], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[13, 14], [14, 14], [15, 14], [14, 15], [14, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 7], [7, 7], [6, 8], [7, 8], [6, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 11], [2, 12], [3, 12], [4, 12], [3, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 3], [8, 4], [9, 4], [9, 5], [10, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "right", "down", "right", "right", "down", "right", "right", "down", "down"]}], [{"image": "training_data/easy/boards/boards1976/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1976/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1976/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1976/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1976/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1976/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1976/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1976/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1976/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1976/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1976/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 3], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[4, 2], [4, 3], [5, 3], [5, 4], [6, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[11, 15], [12, 15], [12, 16], [12, 17], [13, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 5], [16, 5], [17, 5], [16, 6], [16, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 11], [3, 11], [2, 12], [3, 12], [2, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "up", "up", "up", "left", "up", "left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards3550/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3550/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3550/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3550/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3550/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3550/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3550/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3550/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3550/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 2], "target_shape": "Z", "target_color": "magenta", "info": [{"piece_grids": [[9, 1], [10, 1], [10, 2], [10, 3], [11, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 15], [1, 16], [2, 16], [2, 17], [3, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 12], [7, 12], [5, 13], [6, 13], [7, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 13], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 9], [13, 9], [14, 9], [13, 10], [13, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "right", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards533/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards533/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards533/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards533/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards533/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards533/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards533/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 15], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[9, 14], [10, 14], [9, 15], [10, 15], [9, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 15], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 1], [4, 1], [5, 1], [4, 2], [4, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[14, 8], [13, 9], [14, 9], [15, 9], [14, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 5], [2, 6], [3, 6], [3, 7], [4, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards2754/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2754/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2754/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2754/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 8], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[11, 7], [10, 8], [11, 8], [12, 8], [11, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 1], [8, 1], [7, 2], [8, 2], [7, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [7, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 15], [16, 15], [17, 15], [16, 16], [16, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "up"]}], [{"image": "training_data/easy/boards/boards2361/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2361/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2361/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2361/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2361/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2361/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2361/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 13], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[10, 12], [10, 13], [11, 13], [11, 14], [12, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 13], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 1], [4, 1], [2, 2], [3, 2], [4, 2]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [3, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 5], [16, 5], [15, 6], [16, 6], [15, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 8], [7, 8], [7, 9], [7, 10], [8, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "down", "down", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards3208/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3208/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3208/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3208/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3208/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3208/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3208/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3208/step_7.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3208/step_8.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 14], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[11, 13], [12, 13], [12, 14], [12, 15], [13, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[9, 10], [10, 10], [9, 11], [10, 11], [9, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 5], [15, 6], [16, 6], [16, 7], [17, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 1], [6, 1], [7, 1], [6, 2], [6, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "right", "down", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards1205/step_0.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1205/step_1.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1205/step_2.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1205/step_3.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1205/step_4.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1205/step_5.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1205/step_6.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1205/step_7.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1205/step_8.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1205/step_9.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1205/step_10.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1205/step_11.png", "prompt": "You are at the black dot in the board. The target is the red U piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 3], "target_shape": "U", "target_color": "red", "info": [{"piece_grids": [[13, 2], [15, 2], [13, 3], [14, 3], [15, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 10], [2, 10], [2, 11], [2, 12], [3, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 7], [6, 8], [7, 8], [8, 8], [7, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[11, 13], [12, 13], [11, 14], [12, 14], [11, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "up", "right", "right", "up", "up", "up", "right", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards2689/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2689/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2689/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2689/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2689/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2689/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2689/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2689/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2689/step_8.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2689/step_9.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2689/step_10.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2689/step_11.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2689/step_12.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2689/step_13.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 3], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[2, 2], [1, 3], [2, 3], [3, 3], [2, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 8], [12, 9], [13, 9], [13, 10], [14, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 11], [9, 11], [7, 12], [8, 12], [9, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 5], [16, 5], [17, 5], [16, 6], [16, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "left", "left", "left", "left", "up", "up", "up", "left", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards1163/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1163/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1163/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1163/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1163/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1163/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1163/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1163/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1163/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 7], "target_shape": "T", "target_color": "magenta", "info": [{"piece_grids": [[2, 6], [3, 6], [4, 6], [3, 7], [3, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 1], [8, 2], [9, 2], [10, 2], [9, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 15], [16, 15], [15, 16], [16, 16], [15, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 11], [13, 11], [11, 12], [12, 12], [13, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "up", "left", "left", "left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards725/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards725/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards725/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards725/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards725/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards725/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards725/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards725/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards725/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards725/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards725/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards725/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 4], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[2, 3], [3, 3], [4, 3], [3, 4], [3, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[16, 7], [15, 8], [16, 8], [17, 8], [16, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 14], [8, 14], [8, 15], [8, 16], [9, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 11], [13, 11], [12, 12], [13, 12], [12, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "left", "up", "up", "left", "left", "left", "up", "up", "up", "left"]}], [{"image": "training_data/easy/boards/boards2086/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2086/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2086/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2086/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2086/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2086/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2086/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2086/step_7.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2086/step_8.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2086/step_9.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 6], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[14, 5], [14, 6], [15, 6], [15, 7], [16, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 14], [9, 14], [7, 15], [8, 15], [9, 15]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 8], [2, 8], [2, 9], [2, 10], [3, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[5, 11], [6, 11], [5, 12], [6, 12], [5, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "up", "right", "right", "right", "right", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards1904/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1904/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1904/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1904/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1904/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1904/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1904/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 11], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[4, 10], [4, 11], [5, 11], [5, 12], [6, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 15], [14, 15], [14, 16], [14, 17], [15, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[9, 2], [8, 3], [9, 3], [10, 3], [9, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 6], [2, 6], [3, 6], [2, 7], [2, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 7], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["left", "down", "down", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards580/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards580/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards580/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards580/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards580/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 8], "target_shape": "P", "target_color": "magenta", "info": [{"piece_grids": [[6, 7], [7, 7], [6, 8], [7, 8], [6, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 2], [11, 2], [9, 3], [10, 3], [11, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 12], [16, 12], [16, 13], [16, 14], [17, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[1, 15], [1, 16], [2, 16], [2, 17], [3, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["left", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards28/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards28/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards28/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 8], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[8, 7], [9, 7], [8, 8], [9, 8], [8, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 2], [16, 2], [16, 3], [16, 4], [17, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[11, 14], [11, 15], [12, 15], [12, 16], [13, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[5, 10], [4, 11], [5, 11], [6, 11], [5, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left"]}], [{"image": "training_data/easy/boards/boards1547/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1547/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1547/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1547/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1547/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1547/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1547/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1547/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1547/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1547/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1547/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1547/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1547/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 2], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[3, 1], [5, 1], [3, 2], [4, 2], [5, 2]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 14], [8, 14], [8, 15], [8, 16], [9, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 9], [15, 10], [16, 10], [16, 11], [17, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[10, 5], [11, 5], [12, 5], [11, 6], [11, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "up", "up", "up", "left", "left", "up", "up", "left", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards1770/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1770/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1770/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1770/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 11], "target_shape": "U", "target_color": "magenta", "info": [{"piece_grids": [[9, 10], [11, 10], [9, 11], [10, 11], [11, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 1], [13, 2], [14, 2], [15, 2], [14, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 13], [1, 14], [2, 14], [2, 15], [3, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 5], [6, 5], [6, 6], [6, 7], [7, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "right"]}], [{"image": "training_data/easy/boards/boards2002/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2002/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2002/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2002/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2002/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 6], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[9, 5], [9, 6], [10, 6], [10, 7], [11, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 14], [6, 14], [7, 14], [6, 15], [6, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 9], [15, 9], [14, 10], [15, 10], [14, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [1, 2], [2, 2], [3, 2], [2, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["up", "right", "up", "up"]}], [{"image": "training_data/easy/boards/boards2299/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2299/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2299/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2299/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 7], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[9, 6], [9, 7], [10, 7], [10, 8], [11, 8]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 15], [15, 15], [16, 15], [15, 16], [15, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [15, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[4, 10], [5, 10], [4, 11], [5, 11], [4, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[7, 3], [6, 4], [7, 4], [8, 4], [7, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "up", "up"]}], [{"image": "training_data/easy/boards/boards2622/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2622/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2622/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2622/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2622/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2622/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2622/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2622/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 13], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[6, 12], [5, 13], [6, 13], [7, 13], [6, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 3], [16, 3], [17, 3], [16, 4], [16, 5]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[1, 6], [3, 6], [1, 7], [2, 7], [3, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 9], [13, 9], [12, 10], [13, 10], [12, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 10], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "left", "down", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards334/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards334/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards334/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards334/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards334/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards334/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 9], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[4, 8], [5, 8], [4, 9], [5, 9], [4, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 12], [11, 12], [12, 12], [11, 13], [11, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[6, 15], [6, 16], [7, 16], [7, 17], [8, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 4], [13, 5], [14, 5], [15, 5], [14, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1379/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1379/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1379/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1379/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1379/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1379/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1379/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1379/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 15], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[7, 14], [9, 14], [7, 15], [8, 15], [9, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [8, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 1], [2, 1], [2, 2], [2, 3], [3, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[11, 9], [10, 10], [11, 10], [12, 10], [11, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 5], [6, 5], [5, 6], [6, 6], [5, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [5, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down", "down", "down", "down", "down", "left"]}], [{"image": "training_data/easy/boards/boards1426/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1426/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1426/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1426/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1426/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1426/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1426/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 12], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[11, 11], [13, 11], [11, 12], [12, 12], [13, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [12, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 4], [10, 4], [9, 5], [10, 5], [9, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 5], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[5, 14], [6, 14], [7, 14], [6, 15], [6, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 8], [1, 9], [2, 9], [3, 9], [2, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "down", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards1933/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1933/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1933/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1933/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1933/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1933/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1933/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1933/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1933/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1933/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1933/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1933/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 15], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[3, 14], [3, 15], [4, 15], [4, 16], [5, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[11, 7], [13, 7], [11, 8], [12, 8], [13, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 3], [8, 3], [9, 3], [8, 4], [8, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 11], [16, 11], [16, 12], [16, 13], [17, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "down", "down", "down", "down", "left", "down", "left", "left", "down", "left"]}], [{"image": "training_data/easy/boards/boards2057/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2057/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2057/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2057/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2057/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2057/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2057/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2057/step_7.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 7], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[3, 6], [3, 7], [4, 7], [4, 8], [5, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 15], [10, 15], [8, 16], [9, 16], [10, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 10], [15, 10], [15, 11], [15, 12], [16, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[12, 2], [13, 2], [12, 3], [13, 3], [12, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [12, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["up", "left", "up", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards3232/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3232/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3232/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3232/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3232/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3232/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3232/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3232/step_7.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3232/step_8.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3232/step_9.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3232/step_10.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 2], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[5, 1], [6, 1], [6, 2], [6, 3], [7, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[9, 5], [10, 5], [9, 6], [10, 6], [9, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 14], [15, 14], [16, 14], [15, 15], [15, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[11, 11], [13, 11], [11, 12], [12, 12], [13, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "up", "up", "left", "up", "up", "left", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards1948/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1948/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1948/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1948/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1948/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1948/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1948/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1948/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1948/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1948/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1948/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 15], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[4, 14], [4, 15], [5, 15], [5, 16], [6, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[13, 9], [15, 9], [13, 10], [14, 10], [15, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 10], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 3], [8, 3], [9, 3], [8, 4], [8, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 4], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 6], [12, 6], [11, 7], [12, 7], [11, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "left", "left", "down", "left", "down", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards3486/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3486/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3486/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3486/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3486/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3486/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3486/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3486/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3486/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 11], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[14, 10], [15, 10], [15, 11], [15, 12], [16, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 13], [3, 13], [2, 14], [3, 14], [2, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 3], [5, 4], [6, 4], [6, 5], [7, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[10, 7], [9, 8], [10, 8], [11, 8], [10, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}], "path": ["down", "down", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2628/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2628/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2628/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2628/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2628/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2628/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[8, 12], [7, 13], [8, 13], [9, 13], [8, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 5], [3, 5], [2, 6], [3, 6], [2, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 2], [13, 2], [14, 2], [13, 3], [13, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[15, 9], [15, 10], [16, 10], [16, 11], [17, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards1643/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1643/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1643/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1643/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1643/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1643/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1643/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1643/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 6], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[12, 5], [14, 5], [12, 6], [13, 6], [14, 6]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 13], [7, 13], [7, 14], [7, 15], [8, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 2], [17, 2], [16, 3], [17, 3], [16, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[4, 9], [3, 10], [4, 10], [5, 10], [4, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "right", "up", "up", "right", "up"]}], [{"image": "training_data/easy/boards/boards3431/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3431/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3431/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3431/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3431/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 10], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[11, 9], [12, 9], [12, 10], [12, 11], [13, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [12, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[5, 6], [6, 6], [5, 7], [6, 7], [5, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [5, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 1], [7, 2], [8, 2], [9, 2], [8, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 14], [3, 14], [1, 15], [2, 15], [3, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["right", "right", "right", "down"]}], [{"image": "training_data/easy/boards/boards2376/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2376/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2376/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2376/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2376/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2376/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2376/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2376/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2376/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2376/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2376/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2376/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2376/step_12.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2376/step_13.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [15, 2], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[14, 1], [14, 2], [15, 2], [15, 3], [16, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [15, 2], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 10], [10, 10], [11, 10], [10, 11], [10, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 4], [4, 4], [2, 5], [3, 5], [4, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 5], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[6, 7], [7, 7], [6, 8], [7, 8], [6, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [6, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "up", "right", "up", "up", "up", "right", "up", "right", "up", "right", "right", "up"]}], [{"image": "training_data/easy/boards/boards3131/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3131/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3131/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3131/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3131/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3131/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3131/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3131/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 11], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[3, 10], [4, 10], [4, 11], [4, 12], [5, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 1], [15, 1], [13, 2], [14, 2], [15, 2]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 15], [10, 15], [9, 16], [10, 16], [9, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [9, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[9, 0], [10, 0], [11, 0], [10, 1], [10, 2]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 1], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["left", "left", "left", "left", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards1456/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1456/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1456/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1456/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1456/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1456/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1456/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1456/step_7.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1456/step_8.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1456/step_9.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1456/step_10.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 3], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[4, 2], [6, 2], [4, 3], [5, 3], [6, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[11, 15], [12, 15], [12, 16], [12, 17], [13, 17]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [12, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 5], [16, 5], [17, 5], [16, 6], [16, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 11], [3, 11], [2, 12], [3, 12], [2, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "left", "up", "up", "up", "left", "up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards2815/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2815/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2815/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2815/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2815/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2815/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2815/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2815/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2815/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2815/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 14], "target_shape": "X", "target_color": "cyan", "info": [{"piece_grids": [[13, 13], [12, 14], [13, 14], [14, 14], [13, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 5], [7, 5], [7, 6], [7, 7], [8, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 8], [16, 8], [17, 8], [16, 9], [16, 10]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [3, 1], [2, 2], [3, 2], [2, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "down", "down", "down", "right", "right", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards423/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards423/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards423/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards423/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards423/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards423/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards423/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards423/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards423/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 6], "target_shape": "P", "target_color": "cyan", "info": [{"piece_grids": [[4, 5], [5, 5], [4, 6], [5, 6], [4, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[6, 10], [7, 10], [8, 10], [7, 11], [7, 12]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 1], [11, 1], [11, 2], [11, 3], [12, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 13], [15, 13], [13, 14], [14, 14], [15, 14]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "up", "left", "up", "up", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1359/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1359/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1359/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1359/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1359/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 8], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[5, 7], [7, 7], [5, 8], [6, 8], [7, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [6, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[10, 2], [9, 3], [10, 3], [11, 3], [10, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 13], [12, 14], [13, 14], [13, 15], [14, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 14], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[16, 10], [17, 10], [16, 11], [17, 11], [16, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards318/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards318/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards318/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards318/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards318/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards318/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards318/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards318/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards318/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow P piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 8], "target_shape": "P", "target_color": "yellow", "info": [{"piece_grids": [[16, 7], [17, 7], [16, 8], [17, 8], [16, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 14], [7, 15], [8, 15], [8, 16], [9, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[10, 1], [11, 1], [11, 2], [11, 3], [12, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 11], [1, 12], [2, 12], [3, 12], [2, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "right", "right", "up", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2080/step_0.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2080/step_1.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2080/step_2.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2080/step_3.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2080/step_4.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2080/step_5.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2080/step_6.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2080/step_7.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2080/step_8.png", "prompt": "You are at the black dot in the board. The target is the green W piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 15], "target_shape": "W", "target_color": "green", "info": [{"piece_grids": [[6, 14], [6, 15], [7, 15], [7, 16], [8, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 10], [5, 10], [3, 11], [4, 11], [5, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 1], [9, 2], [10, 2], [11, 2], [10, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [10, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 5], [14, 5], [13, 6], [14, 6], [13, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "left", "down", "down", "down", "left", "down", "down"]}], [{"image": "training_data/easy/boards/boards3408/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3408/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3408/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3408/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3408/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3408/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan Z piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 13], "target_shape": "Z", "target_color": "cyan", "info": [{"piece_grids": [[7, 12], [8, 12], [8, 13], [8, 14], [9, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [8, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[2, 5], [3, 5], [2, 6], [3, 6], [2, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [2, 6], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 2], [13, 2], [14, 2], [13, 3], [13, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [13, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[15, 9], [15, 10], [16, 10], [16, 11], [17, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "down", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards2387/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2387/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2387/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2387/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2387/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2387/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2387/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2387/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 7], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[3, 6], [3, 7], [4, 7], [4, 8], [5, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 7], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 14], [11, 14], [12, 14], [11, 15], [11, 16]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 2], [16, 2], [14, 3], [15, 3], [16, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[8, 11], [9, 11], [8, 12], [9, 12], [8, 13]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "left", "left", "left", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards189/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards189/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards189/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 9], "target_shape": "P", "target_color": "blue", "info": [{"piece_grids": [[11, 8], [12, 8], [11, 9], [12, 9], [11, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 12], [14, 12], [14, 13], [14, 14], [15, 14]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 13], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[3, 5], [2, 6], [3, 6], [4, 6], [3, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[5, 15], [6, 15], [7, 15], [6, 16], [6, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}], "path": ["right", "right"]}], [{"image": "training_data/easy/boards/boards1981/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1981/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1981/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1981/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1981/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1981/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1981/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1981/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1981/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1981/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1981/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1981/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1981/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1981/step_13.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1981/step_14.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 16], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[1, 15], [1, 16], [2, 16], [2, 17], [3, 17]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [2, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 12], [6, 12], [7, 12], [6, 13], [6, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[13, 8], [14, 8], [13, 9], [14, 9], [13, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[15, 1], [16, 1], [16, 2], [16, 3], [17, 3]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "down", "left", "left", "down", "left", "down", "left", "left", "down", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards194/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards194/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards194/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards194/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards194/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards194/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards194/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards194/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards194/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards194/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards194/step_10.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards194/step_11.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards194/step_12.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards194/step_13.png", "prompt": "You are at the black dot in the board. The target is the blue P piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 2], "target_shape": "P", "target_color": "blue", "info": [{"piece_grids": [[3, 1], [4, 1], [3, 2], [4, 2], [3, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[12, 9], [12, 10], [13, 10], [13, 11], [14, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 14], [8, 14], [8, 15], [8, 16], [9, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[15, 5], [16, 5], [17, 5], [16, 6], [16, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "left", "up", "up", "left", "left", "up", "up", "left", "up", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards1838/step_0.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1838/step_1.png", "prompt": "You are at the black dot in the board. The target is the red W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 9], "target_shape": "W", "target_color": "red", "info": [{"piece_grids": [[7, 8], [7, 9], [8, 9], [8, 10], [9, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [8, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 14], [3, 14], [3, 15], [3, 16], [4, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[11, 3], [12, 3], [11, 4], [12, 4], [11, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 4], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[16, 11], [15, 12], [16, 12], [17, 12], [16, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 12], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}], "path": ["left"]}], [{"image": "training_data/easy/boards/boards2331/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2331/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2331/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2331/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2331/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2331/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2331/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2331/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2331/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2331/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2331/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 4], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[3, 3], [3, 4], [4, 4], [4, 5], [5, 5]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [4, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 13], [16, 13], [15, 14], [16, 14], [15, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[10, 7], [11, 7], [12, 7], [11, 8], [11, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [11, 8], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[6, 10], [8, 10], [6, 11], [7, 11], [8, 11]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "left", "up", "left", "left", "up", "left", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards604/step_0.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards604/step_1.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards604/step_2.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards604/step_3.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards604/step_4.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards604/step_5.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards604/step_6.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [5, 11], "target_shape": "T", "target_color": "red", "info": [{"piece_grids": [[4, 10], [5, 10], [6, 10], [5, 11], [5, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [5, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 15], [15, 15], [13, 16], [14, 16], [15, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[8, 2], [9, 2], [9, 3], [9, 4], [10, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 6], [1, 7], [2, 7], [3, 7], [2, 8]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["left", "down", "left", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards2772/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2772/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2772/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2772/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2772/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 7], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[7, 6], [6, 7], [7, 7], [8, 7], [7, 8]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 15], [17, 15], [15, 16], [16, 16], [17, 16]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[3, 11], [4, 11], [5, 11], [4, 12], [4, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 2], [14, 2], [13, 3], [14, 3], [13, 4]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}], "path": ["up", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards3335/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3335/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3335/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3335/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3335/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3335/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3335/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3335/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3335/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards3335/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow Z piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 14], "target_shape": "Z", "target_color": "yellow", "info": [{"piece_grids": [[12, 13], [13, 13], [13, 14], [13, 15], [14, 15]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [13, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 5], [6, 6], [7, 6], [8, 6], [7, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 8], [16, 8], [17, 8], [16, 9], [16, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [3, 1], [2, 2], [3, 2], [2, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top left"}], "path": ["down", "down", "right", "down", "right", "right", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards2773/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2773/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2773/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2773/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2773/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2773/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2773/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2773/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2773/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 4], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[6, 3], [5, 4], [6, 4], [7, 4], [6, 5]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[13, 14], [15, 14], [13, 15], [14, 15], [15, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[1, 8], [2, 8], [3, 8], [2, 9], [2, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[9, 11], [10, 11], [9, 12], [10, 12], [9, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}], "path": ["up", "up", "up", "left", "up", "left", "up", "left"]}], [{"image": "training_data/easy/boards/boards2423/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2423/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2423/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2423/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2423/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2423/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2423/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2423/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 6], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[13, 5], [12, 6], [13, 6], [14, 6], [13, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 13], [8, 13], [6, 14], [7, 14], [8, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 2], [17, 2], [16, 3], [17, 3], [16, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 9], [4, 9], [4, 10], [4, 11], [5, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "up", "up", "right", "up", "right"]}], [{"image": "training_data/easy/boards/boards1509/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1509/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1509/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1509/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1509/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1509/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1509/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1509/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1509/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 2], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[7, 1], [9, 1], [7, 2], [8, 2], [9, 2]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 9], [3, 9], [3, 10], [3, 11], [4, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[12, 5], [12, 6], [13, 6], [13, 7], [14, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[15, 15], [16, 15], [17, 15], [16, 16], [16, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [16, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "up", "up", "up", "left", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2324/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2324/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2324/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2324/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2324/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2324/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2324/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2324/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2324/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2324/step_9.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2324/step_10.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2324/step_11.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2324/step_12.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2324/step_13.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 15], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[1, 14], [1, 15], [2, 15], [2, 16], [3, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[5, 11], [6, 11], [5, 12], [6, 12], [5, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 12], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[16, 3], [15, 4], [16, 4], [17, 4], [16, 5]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 4], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[7, 7], [8, 7], [8, 8], [8, 9], [9, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}], "path": ["left", "down", "down", "down", "left", "left", "down", "down", "left", "left", "left", "left", "down"]}], [{"image": "training_data/easy/boards/boards1563/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1563/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1563/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1563/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1563/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1563/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1563/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1563/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1563/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1563/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1563/step_10.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1563/step_11.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1563/step_12.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [2, 14], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[1, 13], [3, 13], [1, 14], [2, 14], [3, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [2, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[6, 5], [5, 6], [6, 6], [7, 6], [6, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 6], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[9, 10], [10, 10], [9, 11], [10, 11], [9, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 11], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 2], [13, 3], [14, 3], [14, 4], [15, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top right"}], "path": ["down", "left", "down", "left", "left", "left", "down", "down", "left", "down", "left", "left"]}], [{"image": "training_data/easy/boards/boards789/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards789/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards789/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards789/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards789/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards789/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards789/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue T piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 13], "target_shape": "T", "target_color": "blue", "info": [{"piece_grids": [[6, 12], [7, 12], [8, 12], [7, 13], [7, 14]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [7, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[9, 2], [10, 2], [10, 3], [10, 4], [11, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [10, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 5], [1, 6], [2, 6], [2, 7], [3, 7]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [2, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[13, 15], [15, 15], [13, 16], [14, 16], [15, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["left", "down", "left", "down", "down", "down"]}], [{"image": "training_data/easy/boards/boards1628/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1628/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1628/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1628/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1628/step_4.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1628/step_5.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1628/step_6.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1628/step_7.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1628/step_8.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1628/step_9.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1628/step_10.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1628/step_11.png", "prompt": "You are at the black dot in the board. The target is the cyan U piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 15], "target_shape": "U", "target_color": "cyan", "info": [{"piece_grids": [[13, 14], [15, 14], [13, 15], [14, 15], [15, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [14, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 7], [7, 7], [6, 8], [7, 8], [6, 9]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 11], [3, 11], [4, 11], [3, 12], [3, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 3], [8, 4], [9, 4], [9, 5], [10, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "right", "right", "down", "down", "right", "down", "down", "right", "right"]}], [{"image": "training_data/easy/boards/boards1424/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1424/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1424/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1424/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [8, 7], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[7, 6], [9, 6], [7, 7], [8, 7], [9, 7]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [8, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 9], [14, 9], [15, 9], [14, 10], [14, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [14, 10], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[10, 13], [11, 13], [11, 14], [11, 15], [12, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 14], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 2], [3, 3], [4, 3], [4, 4], [5, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}], "path": ["up", "up", "left"]}], [{"image": "training_data/easy/boards/boards2736/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2736/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2736/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 11], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[9, 10], [8, 11], [9, 11], [10, 11], [9, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [9, 11], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 2], [2, 3], [3, 3], [3, 4], [4, 4]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [3, 3], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[15, 5], [16, 5], [17, 5], [16, 6], [16, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[12, 14], [13, 14], [13, 15], [13, 16], [14, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["down", "down"]}], [{"image": "training_data/easy/boards/boards2176/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2176/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2176/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2176/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2176/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2176/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2176/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow W piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [7, 5], "target_shape": "W", "target_color": "yellow", "info": [{"piece_grids": [[6, 4], [6, 5], [7, 5], [7, 6], [8, 6]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 5], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[10, 10], [11, 10], [11, 11], [11, 12], [12, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 11], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[4, 7], [3, 8], [4, 8], [5, 8], [4, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 13], [17, 13], [15, 14], [16, 14], [17, 14]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "left", "up", "left", "up", "up"]}], [{"image": "training_data/easy/boards/boards2602/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2602/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2602/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2602/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2602/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2602/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2602/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [10, 14], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[10, 13], [9, 14], [10, 14], [11, 14], [10, 15]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [10, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 10], [14, 10], [12, 11], [13, 11], [14, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[16, 6], [17, 6], [16, 7], [17, 7], [16, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 7], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 2], [7, 2], [8, 2], [7, 3], [7, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 3], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "down", "down", "down", "right", "down"]}], [{"image": "training_data/easy/boards/boards2710/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2710/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2710/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2710/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2710/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2710/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2710/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2710/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2710/step_8.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2710/step_9.png", "prompt": "You are at the black dot in the board. The target is the yellow X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [12, 3], "target_shape": "X", "target_color": "yellow", "info": [{"piece_grids": [[12, 2], [11, 3], [12, 3], [13, 3], [12, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [12, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[2, 15], [4, 15], [2, 16], [3, 16], [4, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 16], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[7, 11], [8, 11], [8, 12], [8, 13], [9, 13]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [8, 12], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 8], [16, 8], [17, 8], [16, 9], [16, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "up", "up", "up", "right", "up", "up", "up", "right"]}], [{"image": "training_data/easy/boards/boards2921/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2921/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2921/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2921/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2921/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2921/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta X piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [6, 7], "target_shape": "X", "target_color": "magenta", "info": [{"piece_grids": [[6, 6], [5, 7], [6, 7], [7, 7], [6, 8]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 7], "piece_shape": "X", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 2], [14, 2], [13, 3], [14, 3], [13, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[9, 11], [11, 11], [9, 12], [10, 12], [11, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [10, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 15], [3, 15], [4, 15], [3, 16], [3, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["left", "up", "left", "left", "up"]}], [{"image": "training_data/easy/boards/boards3591/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3591/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3591/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3591/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3591/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3591/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3591/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 8], "target_shape": "Z", "target_color": "magenta", "info": [{"piece_grids": [[13, 7], [14, 7], [14, 8], [14, 9], [15, 9]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 8], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 2], [9, 2], [7, 3], [8, 3], [9, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 3], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[4, 13], [3, 14], [4, 14], [5, 14], [4, 15]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [4, 14], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[10, 10], [10, 11], [11, 11], [11, 12], [12, 12]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [11, 11], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}], "path": ["right", "right", "up", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2644/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2644/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2644/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2644/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2644/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2644/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2644/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the top. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 3], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[9, 2], [8, 3], [9, 3], [10, 3], [9, 4]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 3], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[11, 14], [13, 14], [11, 15], [12, 15], [13, 15]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 15], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 9], [3, 10], [4, 10], [4, 11], [5, 11]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [4, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[15, 6], [16, 6], [16, 7], [16, 8], [17, 8]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["up", "up", "up", "up", "up", "up"]}], [{"image": "training_data/easy/boards/boards2332/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2332/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2332/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2332/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2332/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2332/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2332/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2332/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2332/step_8.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 6], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[13, 5], [13, 6], [14, 6], [14, 7], [15, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[5, 11], [7, 11], [5, 12], [6, 12], [7, 12]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[3, 8], [2, 9], [3, 9], [4, 9], [3, 10]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 9], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 2], [11, 2], [10, 3], [11, 3], [10, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["right", "up", "right", "right", "up", "up", "right", "right"]}], [{"image": "training_data/easy/boards/boards3102/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3102/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3102/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3102/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3102/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3102/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3102/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards3102/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue Z piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 9], "target_shape": "Z", "target_color": "blue", "info": [{"piece_grids": [[15, 8], [16, 8], [16, 9], [16, 10], [17, 10]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 9], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 5], [10, 5], [8, 6], [9, 6], [10, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 11], [12, 12], [13, 12], [13, 13], [14, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[3, 1], [2, 2], [3, 2], [4, 2], [3, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards828/step_0.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards828/step_1.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards828/step_2.png", "prompt": "You are at the black dot in the board. The target is the green T piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 11], "target_shape": "T", "target_color": "green", "info": [{"piece_grids": [[8, 10], [9, 10], [10, 10], [9, 11], [9, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [9, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[6, 15], [7, 15], [6, 16], [7, 16], [6, 17]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [6, 16], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 4], [12, 4], [12, 5], [12, 6], [13, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [12, 5], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[3, 7], [2, 8], [3, 8], [4, 8], [3, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 8], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}], "path": ["down", "down"]}], [{"image": "training_data/easy/boards/boards2244/step_0.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2244/step_1.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2244/step_2.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards2244/step_3.png", "prompt": "You are at the black dot in the board. The target is the cyan W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [11, 8], "target_shape": "W", "target_color": "cyan", "info": [{"piece_grids": [[10, 7], [10, 8], [11, 8], [11, 9], [12, 9]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 1], [14, 1], [14, 2], [14, 3], [15, 3]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [14, 2], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 15], [6, 15], [7, 15], [6, 16], [6, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[1, 10], [3, 10], [1, 11], [2, 11], [3, 11]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [2, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "up"]}], [{"image": "training_data/easy/boards/boards65/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards65/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards65/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards65/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards65/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards65/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards65/step_6.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards65/step_7.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards65/step_8.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards65/step_9.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards65/step_10.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards65/step_11.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards65/step_12.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 14], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[16, 13], [17, 13], [16, 14], [17, 14], [16, 15]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [16, 14], "piece_shape": "P", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[4, 10], [5, 10], [6, 10], [5, 11], [5, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [5, 11], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 1], [9, 2], [10, 2], [11, 2], [10, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[12, 6], [13, 6], [13, 7], [13, 8], [14, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 7], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["right", "down", "right", "right", "down", "down", "right", "down", "right", "right", "down", "right"]}], [{"image": "training_data/easy/boards/boards1340/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1340/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1340/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1340/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1340/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1340/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1340/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1340/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1340/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards1340/step_9.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 6], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[2, 5], [4, 5], [2, 6], [3, 6], [4, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 6], "piece_shape": "U", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 9], [10, 10], [11, 10], [11, 11], [12, 11]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [11, 10], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[14, 14], [15, 14], [15, 15], [15, 16], [16, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [15, 15], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[7, 1], [8, 1], [7, 2], [8, 2], [7, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [7, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}], "path": ["up", "left", "left", "left", "left", "up", "up", "left", "left"]}], [{"image": "training_data/easy/boards/boards2668/step_0.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2668/step_1.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2668/step_2.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2668/step_3.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2668/step_4.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2668/step_5.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2668/step_6.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2668/step_7.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2668/step_8.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2668/step_9.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2668/step_10.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2668/step_11.png", "prompt": "You are at the black dot in the board. The target is the green X piece located at the bottom right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [14, 15], "target_shape": "X", "target_color": "green", "info": [{"piece_grids": [[14, 14], [13, 15], [14, 15], [15, 15], [14, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [14, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[6, 7], [7, 7], [6, 8], [7, 8], [6, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [6, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[2, 11], [3, 11], [4, 11], [3, 12], [3, 13]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[8, 3], [8, 4], [9, 4], [9, 5], [10, 5]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [9, 4], "piece_shape": "W", "piece_rotation": 0, "piece_region": "top"}], "path": ["down", "right", "right", "right", "right", "down", "down", "down", "down", "down", "right"]}], [{"image": "training_data/easy/boards/boards74/step_0.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards74/step_1.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards74/step_2.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards74/step_3.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards74/step_4.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards74/step_5.png", "prompt": "You are at the black dot in the board. The target is the red P piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 9], "target_shape": "P", "target_color": "red", "info": [{"piece_grids": [[4, 8], [5, 8], [4, 9], [5, 9], [4, 10]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [4, 9], "piece_shape": "P", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 12], [11, 12], [12, 12], [11, 13], [11, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [11, 13], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[6, 15], [6, 16], [7, 16], [7, 17], [8, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[14, 4], [13, 5], [14, 5], [15, 5], [14, 6]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [14, 5], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top right"}], "path": ["left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards2312/step_0.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2312/step_1.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2312/step_2.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2312/step_3.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2312/step_4.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2312/step_5.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2312/step_6.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards2312/step_7.png", "prompt": "You are at the black dot in the board. The target is the magenta W piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 9], "target_shape": "W", "target_color": "magenta", "info": [{"piece_grids": [[15, 8], [15, 9], [16, 9], [16, 10], [17, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 9], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 1], [4, 1], [2, 2], [3, 2], [4, 2]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [3, 2], "piece_shape": "U", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 15], [8, 15], [8, 16], [8, 17], [9, 17]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 16], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[12, 4], [13, 4], [14, 4], [13, 5], [13, 6]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [13, 5], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}], "path": ["right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards3271/step_0.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3271/step_1.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3271/step_2.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3271/step_3.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3271/step_4.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3271/step_5.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3271/step_6.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3271/step_7.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3271/step_8.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards3271/step_9.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards3271/step_10.png", "prompt": "You are at the black dot in the board. The target is the green Z piece located at the top left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [4, 4], "target_shape": "Z", "target_color": "green", "info": [{"piece_grids": [[3, 3], [4, 3], [4, 4], [4, 5], [5, 5]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [4, 4], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[9, 11], [11, 11], [9, 12], [10, 12], [11, 12]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [10, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 7], [14, 7], [13, 8], [14, 8], [13, 9]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 8], "piece_shape": "P", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[15, 14], [15, 15], [16, 15], [16, 16], [17, 16]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [16, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["up", "left", "up", "left", "left", "left", "up", "up", "left", "up"]}], [{"image": "training_data/easy/boards/boards1950/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1950/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue W piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 8], "target_shape": "W", "target_color": "blue", "info": [{"piece_grids": [[8, 7], [8, 8], [9, 8], [9, 9], [10, 9]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 8], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[16, 2], [17, 2], [16, 3], [17, 3], [16, 4]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[12, 14], [13, 14], [14, 14], [13, 15], [13, 16]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 15], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}, {"piece_grids": [[5, 10], [7, 10], [5, 11], [6, 11], [7, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [6, 11], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}], "path": ["up"]}], [{"image": "training_data/easy/boards/boards1542/step_0.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1542/step_1.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1542/step_2.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1542/step_3.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1542/step_4.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1542/step_5.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1542/step_6.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1542/step_7.png", "prompt": "You are at the black dot in the board. The target is the yellow U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 9], "target_shape": "U", "target_color": "yellow", "info": [{"piece_grids": [[15, 8], [17, 8], [15, 9], [16, 9], [17, 9]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 9], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[8, 5], [9, 5], [9, 6], [9, 7], [10, 7]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [9, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[12, 11], [12, 12], [13, 12], [13, 13], [14, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [13, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[3, 1], [2, 2], [3, 2], [4, 2], [3, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top left"}], "path": ["right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards2431/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2431/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2431/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2431/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2431/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2431/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2431/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2431/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 10], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[3, 9], [2, 10], [3, 10], [4, 10], [3, 11]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 10], "piece_shape": "X", "piece_rotation": 0, "piece_region": "left"}, {"piece_grids": [[10, 13], [12, 13], [10, 14], [11, 14], [12, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [11, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[6, 1], [7, 1], [8, 1], [7, 2], [7, 3]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [7, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[15, 5], [16, 5], [16, 6], [16, 7], [17, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 6], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "left", "left", "left", "left", "left", "left"]}], [{"image": "training_data/easy/boards/boards1402/step_0.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1402/step_1.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1402/step_2.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1402/step_3.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1402/step_4.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1402/step_5.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1402/step_6.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards1402/step_7.png", "prompt": "You are at the black dot in the board. The target is the green U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 12], "target_shape": "U", "target_color": "green", "info": [{"piece_grids": [[12, 11], [14, 11], [12, 12], [13, 12], [14, 12]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [13, 12], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[2, 2], [3, 2], [3, 3], [3, 4], [4, 4]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [3, 3], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "top left"}, {"piece_grids": [[7, 5], [7, 6], [8, 6], [8, 7], [9, 7]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 6], "piece_shape": "W", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[15, 15], [16, 15], [17, 15], [16, 16], [16, 17]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [16, 16], "piece_shape": "T", "piece_rotation": 0, "piece_region": "bottom right"}], "path": ["down", "right", "right", "right", "right", "down", "down"]}], [{"image": "training_data/easy/boards/boards2449/step_0.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2449/step_1.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2449/step_2.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2449/step_3.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2449/step_4.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2449/step_5.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2449/step_6.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2449/step_7.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2449/step_8.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2449/step_9.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2449/step_10.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "left"}, {"image": "training_data/easy/boards/boards2449/step_11.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "down"}, {"image": "training_data/easy/boards/boards2449/step_12.png", "prompt": "You are at the black dot in the board. The target is the red X piece located at the bottom left. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [3, 15], "target_shape": "X", "target_color": "red", "info": [{"piece_grids": [[3, 14], [2, 15], [3, 15], [4, 15], [3, 16]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [3, 15], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}, {"piece_grids": [[9, 8], [10, 8], [11, 8], [10, 9], [10, 10]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [10, 9], "piece_shape": "T", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[7, 1], [8, 1], [7, 2], [8, 2], [7, 3]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [7, 2], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[14, 11], [14, 12], [15, 12], [15, 13], [16, 13]], "piece_colour": "cyan", "colour_value": [0, 0, 255], "start_position": [15, 12], "piece_shape": "W", "piece_rotation": 0, "piece_region": "right"}], "path": ["down", "left", "left", "down", "left", "down", "down", "left", "left", "down", "left", "down"]}], [{"image": "training_data/easy/boards/boards1371/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1371/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1371/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the center. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [9, 7], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[8, 6], [10, 6], [8, 7], [9, 7], [10, 7]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [9, 7], "piece_shape": "U", "piece_rotation": 0, "piece_region": "center"}, {"piece_grids": [[13, 1], [14, 1], [15, 1], [14, 2], [14, 3]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [14, 2], "piece_shape": "T", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[5, 15], [5, 16], [6, 16], [6, 17], [7, 17]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [6, 16], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[3, 12], [2, 13], [3, 13], [4, 13], [3, 14]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [3, 13], "piece_shape": "X", "piece_rotation": 0, "piece_region": "bottom left"}], "path": ["up", "up"]}], [{"image": "training_data/easy/boards/boards1358/step_0.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards1358/step_1.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1358/step_2.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1358/step_3.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1358/step_4.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1358/step_5.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1358/step_6.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1358/step_7.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards1358/step_8.png", "prompt": "You are at the black dot in the board. The target is the blue U piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [16, 8], "target_shape": "U", "target_color": "blue", "info": [{"piece_grids": [[15, 7], [17, 7], [15, 8], [16, 8], [17, 8]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [16, 8], "piece_shape": "U", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[7, 14], [7, 15], [8, 15], [8, 16], [9, 16]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [8, 15], "piece_shape": "W", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[11, 1], [10, 2], [11, 2], [12, 2], [11, 3]], "piece_colour": "magenta", "colour_value": [255, 0, 255], "start_position": [11, 2], "piece_shape": "X", "piece_rotation": 0, "piece_region": "top"}, {"piece_grids": [[1, 11], [2, 11], [3, 11], [2, 12], [2, 13]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [2, 12], "piece_shape": "T", "piece_rotation": 0, "piece_region": "left"}], "path": ["up", "right", "right", "right", "right", "right", "right", "right"]}], [{"image": "training_data/easy/boards/boards603/step_0.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards603/step_1.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards603/step_2.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards603/step_3.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards603/step_4.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "up"}, {"image": "training_data/easy/boards/boards603/step_5.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards603/step_6.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "right"}, {"image": "training_data/easy/boards/boards603/step_7.png", "prompt": "You are at the black dot in the board. The target is the red T piece located at the right. Your task is to move towards the target and grab it. Predict your next move from up, down, left, right, grip.", "ground_truth": "grip"}, {"agent_start_pos": [9, 9], "target_pos": [13, 6], "target_shape": "T", "target_color": "red", "info": [{"piece_grids": [[12, 5], [13, 5], [14, 5], [13, 6], [13, 7]], "piece_colour": "red", "colour_value": [255, 0, 0], "start_position": [13, 6], "piece_shape": "T", "piece_rotation": 0, "piece_region": "right"}, {"piece_grids": [[6, 13], [8, 13], [6, 14], [7, 14], [8, 14]], "piece_colour": "green", "colour_value": [0, 255, 0], "start_position": [7, 14], "piece_shape": "U", "piece_rotation": 0, "piece_region": "bottom"}, {"piece_grids": [[16, 2], [17, 2], [16, 3], [17, 3], [16, 4]], "piece_colour": "yellow", "colour_value": [255, 255, 0], "start_position": [16, 3], "piece_shape": "P", "piece_rotation": 0, "piece_region": "top right"}, {"piece_grids": [[3, 9], [4, 9], [4, 10], [4, 11], [5, 11]], "piece_colour": "blue", "colour_value": [0, 0, 255], "start_position": [4, 10], "piece_shape": "Z", "piece_rotation": 0, "piece_region": "left"}], "path": ["right", "right", "up", "up", "up", "right", "right"]}]]
\ No newline at end of file