egocentric_navigation.json

[{"video": "left/4504_frame52.mp4", "question": "This is a navigation video of an agent following instruction: \"Go up the stairs. Take a left at the top of the stairs. Go into the bedroom on the left. Stop in the doorway.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/2763_frame73.mp4", "question": "This is a navigation video of an agent following instruction: \"Go left through the door.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Stop", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/3858_frame48.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk towards the mirror, and take a right. Walk straight through the open door in front of you. Make a hard left, then another. Wait by the long table.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/6842_frame38.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk straight and then turn left and go between the couch and kitchen counter. Enter the next room and stop at the corner of the first white couch and wait.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Turn left and move forward"}, {"video": "left/4668_frame54.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and exit the room. Leave through the hallway and pass the painting. Turn left and exit the hallway. Turn left and stop.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/2988_frame38.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the bathroom, and walk to the stair landing. Walk down the stairs. Make a hard left just before the radiator. Wait at the buffet in the dining room.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Move forward", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/6632_frame30.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk up the stairs Through the doorway on the left. Make a left in the room and stop before the two manikins.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/1581_frame33.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk to the front of the fireplace.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Stop", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/1194_frame53.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk out of the kitchen, past the oven, past the sofas towards the fireplace. Stop at the doorway facing towards the next sitting room facing the dining room table.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Turn left and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/4597_frame44.mp4", "question": "This is a navigation video of an agent following instruction: \"walk around the sofa then walk straight at the island table.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Turn right and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/5938_frame39.mp4", "question": "This is a navigation video of an agent following instruction: \"Go up the stairs. Turn left and go into the glass doors. Wait at the entrance.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Turn right and move forward", "Move forward"], "answer": "Turn left and move forward"}, {"video": "left/431_frame33.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit this bedroom and travel to the one that is next door.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/4492_frame38.mp4", "question": "This is a navigation video of an agent following instruction: \"Go up the stairs and turn left. Wait at the doorway to the bedroom straight ahead.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/1161_frame34.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the bedroom and go down the hall slightly. Make a left into the last room down the hall. Go into that room and enter into the bathroom on the left hand side.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn right and move forward", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/1090_frame31.mp4", "question": "This is a navigation video of an agent following instruction: \"Continue up the stairs and take a right into the bedroom. In the bedroom take a left and go through the closet, and stop in the bathroom doorway.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Stop", "Move forward"], "answer": "Turn left and move forward"}, {"video": "left/2380_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk out of the room and go past the stairs and into the bedroom that is straight ahead, go to the bathroom doorway and wait there.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/1720_frame34.mp4", "question": "This is a navigation video of an agent following instruction: \"Take a left and exit the closet. Take a left into the bedroom. Take another left into the bathroom and wait there.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/5153_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the bathroom, and take a right. Take the first left and go down a hallway. At the end of the hall take a right into the foyer. Stop before you reach the mirror.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/2639_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk left along the path in the sauna and out the door until you are in the pool room.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn right and move forward", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/1716_frame42.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the stairs and go straight down the hallway. Turn left into the room and wait near the dining table.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Turn right and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/5162_frame56.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the room, and take a right. Walk straight across the hall into the room across. Stop once you are in the room on the rug.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/6753_frame55.mp4", "question": "This is a navigation video of an agent following instruction: \"After you walk up the stairs make a right and walk up the second flight of stairs. Then make a left and walk to the sixth step and wait.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Turn right and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/1420_frame30.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk to the end of the hallway, pass the bathroom and wait by the laundry room.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/3411_frame39.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk left past the island and into the dining room. Turn right through the entryway and then left past the microwave. Stop just inside the doorway.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/2978_frame46.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the hallway and past the sofa. Walk into the dining room area. Stop in front of the island.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/955_frame45.mp4", "question": "This is a navigation video of an agent following instruction: \"Start at the bedroom door. Go out into the hallway. Turn right. Walk past a bathroom on the left. Go into the bedroom door at the end of the hallway on the left.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Turn right and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/4993_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Go out of the bedroom towards some stairs. Pass the stairs and go into a bedroom with some posters of girls on the wall and wait at the doorway to the bathroom.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Turn left and move forward", "Move forward"], "answer": "Turn left and move forward"}, {"video": "left/6626_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and go to the staircase. Go around the stairs to the left and stop in the bedroom.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Turn left and move forward"}, {"video": "left/5123_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"Go past pug and chair into next room. Go straight past table and stop at doorway.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/5261_frame33.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn right and go down the long hall. Turn left toward the bar. Turn right into the kitchen and stop by the fridge.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Stop", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/4871_frame47.mp4", "question": "This is a navigation video of an agent following instruction: \"Go straight up the stairs. Wait at the top.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/6875_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the closet. Turn right. Turn left and then turn left again. Walk toward the large table and chairs then wait there.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Move forward", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/2695_frame39.mp4", "question": "This is a navigation video of an agent following instruction: \"Go to the painting. Go to the electrical receptacle. Go to the NQHG painting. Go to the couch. Go to the footstool.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/1119_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the pool until you get to the bar. Walk up the three stairs and stop in the doorway.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/6550_frame53.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the room, then immediately turn left through the doorway. Wait by the double doors.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/4268_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"With the violin on your right, walk straight down the hallway, past the windows on the right. Turn left at the end and stop in the round doorway.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/4573_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk across the hall, through the bedroom and stop inside the en suite bathroom.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/7276_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk up the stairs to the second landing, then turn left. Wait on the rug, near the doorway.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn right and move forward", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/4631_frame62.mp4", "question": "This is a navigation video of an agent following instruction: \"Go to the rug in front of the outside doorway. Go to the dining room table. Go to the right of the fireplace. Go through the door.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/5148_frame38.mp4", "question": "This is a navigation video of an agent following instruction: \"exit the room then turn hard left, walk forward down the stairs and stop at the front of the bathroom.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/1030_frame47.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the bedroom and go into the den and take your first right into a hallway. A the end of the hallway take a left and go into the sitting room on the right.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Stop", "Move forward"], "answer": "Turn left and move forward"}, {"video": "left/4009_frame48.mp4", "question": "This is a navigation video of an agent following instruction: \"Go forward down the hall. At the end of the hall take a left, and take another left into the sitting room with the mural. Stop once you enter the room.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Turn left and move forward"}, {"video": "left/6164_frame43.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk straight. Pass the fireplace on your right. When you get to the pool table turn left and wait.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Turn left and move forward", "Move forward"], "answer": "Turn left and move forward"}, {"video": "left/5952_frame39.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the room, turn left, and go into the left room at the end of the hall. Proceed through this room and wait by the sink in the adjacent bathroom.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/4448_frame51.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk all the way forward passing all the picture frames on the wall on your left. Enter the corner on your left with the arch layout, and stop there.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Stop", "Turn right and move forward"], "answer": "Turn left and move forward"}, {"video": "left/6843_frame48.mp4", "question": "This is a navigation video of an agent following instruction: \"Go down the hall on the left and into the living room. Go through the door on the right until you reach the treadmill.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Turn left and move forward"}, {"video": "left/1382_frame58.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the stairs towards and past the atrium. Head towards the kitchen and wait in the dining area.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "left/101_frame51.mp4", "question": "This is a navigation video of an agent following instruction: \"Head diagonally through the kitchen, stop at the edge of the kitchen table, turn left and stop in the middle of the room near a vase on the floor.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Move forward", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/6939_frame50.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn left. Turn right when you get to the glass panels. Wait next to the table with the book, purse, and hat on it.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn right and move forward", "Turn left and move forward"], "answer": "Turn left and move forward"}, {"video": "left/4332_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"Go forward toward the windows. Go toward the the far couch, Stop next to the couch, in front of the windows.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Turn left and move forward"}, {"video": "right/4148_frame41.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn left just beyond staircase railing to enter hallway. Walk past painting on the wall. Turn right. Walk into room with circle rug. Wait here.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/1631_frame39.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the art room an take a left. Go past the stairs and take a right into the kitchen Stop in the kitchen with the wine fridge on your left.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn right and move forward", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/1888_frame54.mp4", "question": "This is a navigation video of an agent following instruction: \"Left the living room and make a left then walk past the stairs and make a left to walk down the hall beside the stairs. Make a right at the end of the hall and go into the art room and wait.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Turn left and move forward", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/3272_frame45.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk into the bedroom and turn right. Walk out of the bedroom and to the top of the stairs. Step to the third step down and stop.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Stop", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/6617_frame43.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk toward the railing and right past the stairs. Walk into the living room and turn right. Stop by the end table.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Move forward", "Stop"], "answer": "Turn right and move forward"}, {"video": "right/5138_frame57.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk toward the stairs to the second floor. Go around the stairs into the living room. Take a right and go into the dining room. Stop next to the first chair you pass.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Turn left and move forward", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/2662_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the bar and make a hard right into the walkway next to the jacuzzi. Wait just inside the door.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Stop", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/321_frame42.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk through the archway to the right and into the bathroom. Wait in the room to the right with the toilet.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/7181_frame30.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and go passed the refridgerator. Turn right and go through the door. Stop near the large table.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn right and move forward", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/4051_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk through the doors opposite the bathroom. Walk past the piano and continue past the violin on the wall. Stop in front of table with the large vase.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/3234_frame73.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the dinning room table and enter the hallway. Then turn left and walk into the kitchen. Stand near the oven.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/657_frame58.mp4", "question": "This is a navigation video of an agent following instruction: \"Go towards a dining area passing an area open to the outside to your right, and pass the dining table. Walk towards three sets of doors, and stop before reaching them.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Turn right and move forward", "Stop"], "answer": "Turn right and move forward"}, {"video": "right/3202_frame33.mp4", "question": "This is a navigation video of an agent following instruction: \"Go down the stairs and go into the door on your right. Wait by the toilet.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/1851_frame34.mp4", "question": "This is a navigation video of an agent following instruction: \"walk forward towards the stairs and take a right. Take a right near the bed into the bathroom. Stop in the doorway to the bathroom.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Stop", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/6966_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the bedroom and turn left into the doorway. Walk through the doorway on the right into the closet and proceed to the bathroom. Wait near the towel bar.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/3406_frame51.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the long hallway next to the stairs, past the wine cellar and wait in the bedroom on the right.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/583_frame34.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the hall, take the first right and wait in the short hallway between the two doors.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/3012_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"walk out of the bathroom and down the hall. Turn right at the end of the hallway. Walk past the piano. Stop under the large chandelier by the violin.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Stop", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/3108_frame47.mp4", "question": "This is a navigation video of an agent following instruction: \"exit bedroom to hallway, turn right and enter bathroom. Stop before bathtub.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Stop", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/478_frame38.mp4", "question": "This is a navigation video of an agent following instruction: \"Go through door. Turn right on corner. Turn right on corner again. Stop.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Turn left and move forward", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/1189_frame31.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the bathroom, enter the bedroom, exit the bedroom to the hallway, wait at the top of the stairs.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Move forward", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/1899_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn to the left and walk past the couch and footstool. Go to the hallway and turn right. And walk down past the galley kitchen, towards a table and chairs area. Stop and wait there before you get to the table.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/1383_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the couch and turn left. Walk into the hall and turn right. Walk into the bedroom and stop by the closet.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Move forward", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/3267_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the bathroom and bedroom. Stop in front of the cabinet full of dolls in the hallway.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/2749_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the room and head right. When you get to frame of the teddy bear with balloons that says \"Welcome\" in the wall turn right. Head straight into that room and wait by the bed.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Stop", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/1690_frame80.mp4", "question": "This is a navigation video of an agent following instruction: \"Go past the circle of chairs toward the door. Turn right and go into the hallway. Turn left to go into the kitchen.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Turn right and move forward", "Stop"], "answer": "Turn right and move forward"}, {"video": "right/4787_frame52.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and enter hall on right. Go forward down hallway until you reach flowers. Turn right and enter. Stop once entered.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Stop", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/3875_frame33.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk in the direction of the stairs. Take a right into the room at the end of the stairs. Stop once you are in the dining room.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Move forward", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/5795_frame50.mp4", "question": "This is a navigation video of an agent following instruction: \"Go up the stairs to the next level and go down the hallway. Turn into the room on the right and walk to the center of it. Walk through the two columns on the right and stop in front of the horned animal head on the floor.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn right and move forward", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/5787_frame90.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and exit out the door on the left. Walk all the way down until you reach the green steps. At the steps, turn right and enter the door to the left. Stop once you reach the table.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn right and move forward", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/3669_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Go through the pillars, make a left go through the doorway, make another left towards the stairs, make a right passed the stairs on your right into the doorway and wait.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/3557_frame30.mp4", "question": "This is a navigation video of an agent following instruction: \"Go down the stairs to your right and stop when you get all the way down.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/6595_frame41.mp4", "question": "This is a navigation video of an agent following instruction: \"go past the window on your left and exit the dining room. Enter the kitchen and turn right passing to the left of the island. Go straight and stop between the wall unit and end table.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Stop", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/4365_frame41.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit bathroom. Turn left and walk into the bedroom. Walk towards the bed and turn right. Exit bedroom. Walk down two stairs and stop on the stairs.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/5627_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the hall parallel to the stairs and turn right. Keep walking straight until you enter a room with a couch and leather footstool in the middle. Stop just after stepping into this room.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Turn left and move forward", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/6482_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down steps, walk towards the bed. Walk on the left side of the bed and walk through archway, and stop in the under the archway.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Turn right and move forward"}, {"video": "right/6576_frame65.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the room. Turn left. Go straight until you pass the stairs. Pass the two first chairs one on each side and then turn right. Wait near the glass table with the white chairs.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/4101_frame40.mp4", "question": "This is a navigation video of an agent following instruction: \"Go straight past the oven. Turn left and enter the doorway. Go into the doorway to the right of the painting and wait.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Turn right and move forward"}, {"video": "right/4563_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn right and walk through the kitchen. Go right into the living room and stop by the rug.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Move forward", "Stop"], "answer": "Turn right and move forward"}, {"video": "right/3731_frame47.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn left. Wait in between the table and chairs on the right and the couches.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/4629_frame76.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn left and go through the kitchen. Turn left and walk past the kitchen island. Turn right and walk past the pottery. Wait inside the room on the left next to the table with the flowers.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/6046_frame53.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the bathroom through the door by the toilet and turn right. Turn right into the hallway, walk through the arched opening, and turn right. Stop near the the first black chair at the bar.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/4469_frame47.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn right and go down the hallway then turn left.Pass the chair on the left and turn right into the kitchen. Wait near the fridge.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/1275_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the flight of stairs to the lower level. Walk into the room ahead with the tile mosaic on the floor. Walk around the corner through the open arched door to the right with rug on the ground.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Stop", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/4865_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Go across the room until your infront of the big doors. Turn right and stay next to the left side of the smallest chair in couch area.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/254_frame60.mp4", "question": "This is a navigation video of an agent following instruction: \"leave sitting room and head towards the kitchen, turn right at living room and enter. walk through living room to dinning room and enter. Turn left and head to front door. Exit the house and stop on porch.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Turn right and move forward"}, {"video": "right/239_frame55.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk through the kitchen, enter the dining room, walk to the doorway to the right of the dining room table, wait at the glass table.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Turn right and move forward", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/6339_frame50.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk to the far end of the lobby and wait under the exit sign.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/6828_frame45.mp4", "question": "This is a navigation video of an agent following instruction: \"Head in the direction of the wine cellar on the left, and enter it. Stop behind the table at the far end.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Turn right and move forward", "Move forward"], "answer": "Turn right and move forward"}, {"video": "right/1927_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk to the back of the couch and stop at the pool table.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Stop", "Move forward"], "answer": "Turn right and move forward"}, {"video": "forward/2291_frame31.mp4", "question": "This is a navigation video of an agent following instruction: \"Go passed the table and passed the stairs. Take a right turn and go down the stairs. Stop on the first stair.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Turn left and move forward", "Move forward"], "answer": "Move forward"}, {"video": "forward/5286_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn left and walk down the hallway. Continue to walk straight through the kitchen. Stop next to the white flowers on the left.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Stop", "Move forward"], "answer": "Move forward"}, {"video": "forward/5822_frame38.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn to walk past curio cabinet. Walk towards kitchen. Stop at light switches on stone wall before the kitchen island with sink.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Turn left and move forward", "Move forward"], "answer": "Move forward"}, {"video": "forward/4152_frame34.mp4", "question": "This is a navigation video of an agent following instruction: \"Head straight into the kitche. Go past the island and cabinets. Stop just inside next room.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Turn right and move forward", "Move forward"], "answer": "Move forward"}, {"video": "forward/4801_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk forward through the doorway, turn right, walk to the next doorway and wait.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Move forward"}, {"video": "forward/4474_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk around the table and exit the room. Walk down the first set of stairs. Wait there.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/6077_frame56.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the sitting room and turn left. Take the first turn to the right.Go straight and then turn left at the shelf. Walk into the first door on the right.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Stop", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/5644_frame30.mp4", "question": "This is a navigation video of an agent following instruction: \"Go up the steps. Walk straight until you pass the kitchen. Continue straight until you get to a very large table with chairs.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Turn right and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/1676_frame44.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn left and walk across the hallway. Turn left again and walk across this hallway and wait by the bathtub.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/2751_frame49.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the hallway and pass all the picture frames placed on the right until the end. Turn left to enter another room, and stop there.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn left and move forward", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/6093_frame70.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and go passed the dining room table. Enter the doors next to the red chairs. Go down the hallway and enter the doors on the right.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn right and move forward", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/3484_frame109.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn right and go through the door. Go down the hall way and go up the stairway to the left. Continue up the stairway to the left. Wait by the statue.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Turn left and move forward", "Move forward"], "answer": "Move forward"}, {"video": "forward/5956_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"Go straight toward the stairs. When you get to the stairs on the left. Stop there.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/6911_frame38.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn left at the lamp. Wait next to the beige chairs and the double glass doors.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Move forward"}, {"video": "forward/4613_frame39.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the bathroom and walk down the hall. Wait at the bottom of the stairs, on the first step.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Stop", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/3038_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the bedroom, turn left, walk through the entryway, turn right, walk through the kitchen, then wait in front of the bathroom.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/4156_frame45.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the bedroom. Go to the right towards the kitchen. Stand next to the hallways located after the cabinets.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Move forward", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/15_frame39.mp4", "question": "This is a navigation video of an agent following instruction: \"Go out of the room you're in. Turn left. Go poast the dining room and into the living room. Wait right by the coffee table.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/6157_frame39.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk in the direction of the table at the end of the hall, and take a left. Walk into the office. Stop once you pass the doors.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/3661_frame41.mp4", "question": "This is a navigation video of an agent following instruction: \"Follow the red carpet on the right side of the room. Follow it past the chairs and stop in the door way at the end of the room.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Turn right and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/5887_frame69.mp4", "question": "This is a navigation video of an agent following instruction: \"walk forward and turn left, walk past the table, then wait near the doorway.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Move forward"}, {"video": "forward/7309_frame57.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and exit the room. Once out, turn right and keep walking straight until you enter the room on the end. Stop once you enter the room.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/1679_frame40.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk out through the door. Follow the red carpet to the right of the table. Stop in the next doorway.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/4954_frame55.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the dining table, then turn right into the hall way. Turn left, walk down the hall, and stop when you reach the sofa.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Move forward", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/552_frame38.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn right. Go forward. Turn right at doorway. Go forward. Turn right again. Go forward and stop.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Turn right and move forward", "Move forward"], "answer": "Move forward"}, {"video": "forward/1195_frame30.mp4", "question": "This is a navigation video of an agent following instruction: \"Go up the stairs. Go left toward the couch and walk past it to the chair next to the fireplace.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Move forward"}, {"video": "forward/1539_frame47.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk behind the couch and turn right. Walk forward and wait near the dining table.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/4941_frame52.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk up steps across patio and enter the house through the slider. Walk across living room and stop at kitchen.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Turn right and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/3525_frame52.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the room and head straight then turn right and pass through the kitchen and then turn left and go into the second door on the right and stop by the sink.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn left and move forward", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/6721_frame34.mp4", "question": "This is a navigation video of an agent following instruction: \"Go straight through the large entryway and turn right, go passed the large table and stop in the entryway.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Move forward"}, {"video": "forward/1111_frame58.mp4", "question": "This is a navigation video of an agent following instruction: \"Go to the stairs\r\nGo to the Buck's Country\r\nGo to the footstool\r\nGo to the semiround chair\r\nGo to the right four metres\r\nGo to the pool table.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/3492_frame46.mp4", "question": "This is a navigation video of an agent following instruction: \"Go right and head up the right side of the house. Go up one step then wait there.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Turn right and move forward", "Move forward"], "answer": "Move forward"}, {"video": "forward/4192_frame34.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and go to the left. Turn right and cross through the kitchen. At the end of the hallway stop.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Turn right and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/4903_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Move to the other side of the center pillar, and wait near its far end.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Move forward"}, {"video": "forward/4940_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk forward passing the dining table on it's right side. Exit the room with the table, and stop before you reach the end of the rug on the right.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/2233_frame42.mp4", "question": "This is a navigation video of an agent following instruction: \"walk through the doorway to the left of the small table with two chairs, walk straight down past art on your right, turn left and stop in the doorway.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Move forward", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/6311_frame47.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the dining room. Wait in the doorway on the right.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/3785_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk across the room and stop next to the door going outside, near the stairs.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Turn right and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/2658_frame46.mp4", "question": "This is a navigation video of an agent following instruction: \"Go down the stairs and turn right. Head toward the pool table and when you get to the fireplace wait there.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/2365_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the two urns and turn left. Walk to the entrance of the dining room, straight ahead, and stop under the arch.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Stop", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/7079_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"Go through the door on the right and continue straight. Stop in the next room in front of the bed.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/6671_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk passed the right of the bar, and turn left to walk through the kitchen. Stop once you reach the shelves on your right.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Stop", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/5512_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the end table and turn right into the kitchen. Walk to the left of the island past the oven into the next room and stop.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Stop", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/4634_frame52.mp4", "question": "This is a navigation video of an agent following instruction: \"Go to the kitchen, then to the dining room table inside.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Turn right and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/1836_frame40.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk straight and to the right of the table and through the doorway. Wait in front of the bench.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Move forward", "Turn left and move forward"], "answer": "Move forward"}, {"video": "forward/2877_frame31.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the bedroom and take a right and walk into the pantry. Walk through the pantry and kitchen towards the dining room. Stop before you reach the table.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn left and move forward", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/2371_frame32.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn left and wait by the beaded doorway at the end of the pool.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn left and move forward", "Turn right and move forward"], "answer": "Move forward"}, {"video": "forward/300_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk towards the french doors and turn left. Walk past the kitchen area. Go to the end of the hallway and wait in the archway near the painting.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/4468_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and go through the door. Turn right and continue down the red carpet. Stop at the end of the hall and the door.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Turn right and move forward", "Stop"], "answer": "Move forward"}, {"video": "forward/1879_frame57.mp4", "question": "This is a navigation video of an agent following instruction: \"Go down the hallway to the right. Keep going until the end of the hall, then enter the dining room. Stop a few steps before the dining table.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Turn left and move forward", "Move forward"], "answer": "Move forward"}, {"video": "stop/2282_frame52.mp4", "question": "This is a navigation video of an agent following instruction: \"Staying on the red carpet, go through the ornate doors next to you and continue forward through the next room until just before the carpet path splits off at a forty-five degree angle to the left and a ninety degree angle to the right.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Stop"}, {"video": "stop/2250_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk up the steps and continue on into the massage room and wait there.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Turn left and move forward", "Move forward"], "answer": "Stop"}, {"video": "stop/3122_frame59.mp4", "question": "This is a navigation video of an agent following instruction: \"Go up the stairs and take a right. Enter the doorway into the kitchen and take a left. Walk through the kitchen and take a left into the dining room. Stop in the doorway to the dining room.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Move forward", "Stop"], "answer": "Stop"}, {"video": "stop/5186_frame68.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk up the steps and turn left. Walk toward the front door and turn right at the steps. Walk up the stairs and stop at the top of the steps.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/5876_frame53.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the room and turn left. Stop just before you reach the stairs that go up.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Stop", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/3280_frame61.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit bedroom through the doorway on the far right. Walk directly forward. Take first sharp right at welcome sign. Wait by the table with a handbag on it.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Turn left and move forward", "Move forward"], "answer": "Stop"}, {"video": "stop/5039_frame54.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk into the bedroom toward the fireplace. Go around the fireplace on the left. Stop in the doorway before you reach the rug.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Move forward", "Stop"], "answer": "Stop"}, {"video": "stop/1431_frame57.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn around and walk out of the closet and take a right and walk out of the bedroom. Take a left and walk strauight. Wait in front of the couch.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/5038_frame50.mp4", "question": "This is a navigation video of an agent following instruction: \"Go left past the chairs and up all the stairs. Stay in doorway of first room.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Turn right and move forward", "Move forward"], "answer": "Stop"}, {"video": "stop/155_frame71.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk towards the kitchen area. Keep walking along the kitchen area towards the doorway. Turn right and exit the kitchen area walking across the small living room area and into the bedroom to you left.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/2288_frame59.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the refrigerator and turn left. Walk past the television and step outside through the sliding glass doors. Stop just outside the glass doors.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn right and move forward", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/945_frame31.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the stairs and head into the pool room. Stop at the bottom of the stairs once you are in the pool room.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn right and move forward", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/4027_frame31.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit bedroom through arch doorway. Enter the room to the left. Wait by door.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Stop", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/2527_frame52.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the kitchen, walk down the hall and wait in the exercise room.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn left and move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/4721_frame48.mp4", "question": "This is a navigation video of an agent following instruction: \"Go towards the dining room table. Go out on the entryway and cross to the other side of the hallway beside the entryway.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/6766_frame49.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn to go past the stairs and straight down the hallway towards the bathroom. Stop when reaching the double doors.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/2141_frame54.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit closet to hallway, go passed to vases and enter room with wine. Stop at wine.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn left and move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/3353_frame39.mp4", "question": "This is a navigation video of an agent following instruction: \"Go straight down the hall and go up the stairs. Stop at the 3rd step.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/2341_frame55.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the refrigerator and out of the kitchen. Enter the second bedroom on the right. Stop near the foot of the bed with a gray comforter.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Move forward", "Stop"], "answer": "Stop"}, {"video": "stop/4749_frame46.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk out of the bedroom and down the hallway past the little living area on the right. Walk into the door into the next bedroom.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Stop", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/4297_frame63.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn left and walk straight across the kitchen and through the storage area. Once out, turn right and then enter the last room on the left and stop in the doorway.\" What is the next action it should take?", "candidates": ["Stop", "Move forward", "Turn right and move forward", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/2782_frame56.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk straight along the hallway and turn left on the first corner. Enter the room and walk until you see the purple sofa on the left side, and stop right in front of the animal showpiece placed on the floor.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Move forward", "Turn left and move forward", "Stop"], "answer": "Stop"}, {"video": "stop/548_frame43.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the hallway and wait near the end, next to the mirror and side table.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/2501_frame45.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn right and go up the wood stairs. At the top walk forward and turn right. Then walk halfway up the stairs covered in carpet and stop.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Stop"}, {"video": "stop/267_frame34.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the bathroom and walk straight into the hallway. Go straight and enter the office in the second door on your left.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Turn right and move forward", "Stop"], "answer": "Stop"}, {"video": "stop/2690_frame26.mp4", "question": "This is a navigation video of an agent following instruction: \"Go to exit of writing room. Stop between pillars.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Stop"}, {"video": "stop/5840_frame44.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn right and go down the hallway. Pass the couches and then turn right. Wait near the coffee table.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Turn left and move forward", "Move forward"], "answer": "Stop"}, {"video": "stop/4943_frame48.mp4", "question": "This is a navigation video of an agent following instruction: \"Stand with the windows facing outdoors on your left and walk down the hallway. When you get to the white planter at the end, turn left. Stop in the middle of the room on the circular floor mosaic.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Stop", "Move forward"], "answer": "Stop"}, {"video": "stop/3529_frame63.mp4", "question": "This is a navigation video of an agent following instruction: \"With the sliding glass door behind you, head out of the room and take a quick dog leg to the left. Go past the kitchen, then turn right and stand between the kitchen and the couch.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Stop"}, {"video": "stop/6893_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn right down the hallway and enter the last room on the left. Stop after entering the bedroom.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Stop"}, {"video": "stop/573_frame36.mp4", "question": "This is a navigation video of an agent following instruction: \"Go through the door on the right, then turn left and walk down the hallway. Walk just past the table and then stop.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Stop", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/5883_frame45.mp4", "question": "This is a navigation video of an agent following instruction: \"Stand with the bed behind you and walk straight down the red carpet. Go into the room with the tile floor and stop after a few steps closer to the middle of the room.\" What is the next action it should take?", "candidates": ["Stop", "Turn right and move forward", "Move forward", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/2564_frame43.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk down the path with an informational sign on your right. Continue straight through the doorway and stop after a couple of feet, just before another informational sign on the right.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Move forward", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/3274_frame61.mp4", "question": "This is a navigation video of an agent following instruction: \"Go through the bathroom and into the kitchenette. Go past the counter and out the open door. Head down the hallway and stop right past the open book.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Stop", "Move forward", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/2834_frame31.mp4", "question": "This is a navigation video of an agent following instruction: \"Step out of the shower and turn left to leave the bathroom. Continue straight ahead into the bedroom to wait by the bed.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/4828_frame68.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk forward and turn right towards the three stairs to enter the house. Walk forward towards the glass door on the very left corner, and stop there.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn right and move forward", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/6822_frame37.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk around the bed to the right and into the hallway. Wait at the end of the hallway.\" What is the next action it should take?", "candidates": ["Move forward", "Stop", "Turn right and move forward", "Turn left and move forward"], "answer": "Stop"}, {"video": "stop/5873_frame35.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk straight and then turn left. Turn right and wait near the door.\" What is the next action it should take?", "candidates": ["Move forward", "Turn left and move forward", "Turn right and move forward", "Stop"], "answer": "Stop"}, {"video": "stop/384_frame77.mp4", "question": "This is a navigation video of an agent following instruction: \"Go down the stairs to the next floor. On the next floor take another right and go down the first section of stairs and stop on the landing in front of the room with the radiator visible.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/2184_frame57.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit sitting room, turn left and go up stairs. At landing turn left and go up stairs. Stop at top of stairs.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Stop"}, {"video": "stop/2905_frame30.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the bathroom and turn right. Continue straight down the hall until you reach the kitchen.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Stop", "Move forward"], "answer": "Stop"}, {"video": "stop/3856_frame53.mp4", "question": "This is a navigation video of an agent following instruction: \"Turn right and enter the kitchen. Wait in the doorway next to the oven.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Stop", "Move forward"], "answer": "Stop"}, {"video": "stop/2071_frame54.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk past the sink, and through the open door. Make a hard left at the closed brown and frosted glass french doors, and then a hard right.. Walk past the kitchen, and wait near the sofa end table.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Move forward", "Stop", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/2599_frame38.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk to the left of the stairs and stop next to the nearest of the two beds.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Stop", "Move forward", "Turn right and move forward"], "answer": "Stop"}, {"video": "stop/6057_frame41.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the toilet room and then exit the tub room and go through the hallway but turn right into the first archway on the right, take a step into the closet and stop.\" What is the next action it should take?", "candidates": ["Stop", "Turn left and move forward", "Turn right and move forward", "Move forward"], "answer": "Stop"}, {"video": "stop/5403_frame50.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the sitting room and go left. Take a right before you enter the foyer. Go straight into the office. Stop on the rug in front of the bookcase.\" What is the next action it should take?", "candidates": ["Turn right and move forward", "Turn left and move forward", "Stop", "Move forward"], "answer": "Stop"}, {"video": "stop/5333_frame59.mp4", "question": "This is a navigation video of an agent following instruction: \"Leave the sitting room and go left. Take a right before you enter the foyer. Take a right before you enter the office area. Enter the bathroom on the left.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Turn left and move forward", "Stop"], "answer": "Stop"}, {"video": "stop/6336_frame47.mp4", "question": "This is a navigation video of an agent following instruction: \"Position yourself so that the red carpet leads to a bed with curtains. Walk straight towards this bed and stop in the doorway of the room and then stop.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Turn left and move forward", "Stop"], "answer": "Stop"}, {"video": "stop/1597_frame58.mp4", "question": "This is a navigation video of an agent following instruction: \"Walk across dining room, turn slight right and enter set of double doors, stop in doorway.\" What is the next action it should take?", "candidates": ["Turn left and move forward", "Turn right and move forward", "Move forward", "Stop"], "answer": "Stop"}, {"video": "stop/3822_frame52.mp4", "question": "This is a navigation video of an agent following instruction: \"Exit the kitchen and wait in the sitting room, near the loveseat.\" What is the next action it should take?", "candidates": ["Move forward", "Turn right and move forward", "Turn left and move forward", "Stop"], "answer": "Stop"}]