MDP	Returns
Random policy	Optimal policy	PPO	DQN	GORP (5M)	(100M)
\( \text{ALIEN}_{10} \)	74.74	160	160	160	160	160
\( \text{AMIDAR}_{20} \)	6.77	109	68	68	65	68
\( \text{ASSAULT}_{10} \)	8.66	126	126	126	126	126
\( \text{ASTERIX}_{10} \)	99.24	400	400	400	400	400
\( \text{ASTEROIDSS}_{10} \)	15.08	320	170	190	220	220
\( \text{ATLANTIS}_{10} \)	24.22	200	200	200	200	200
\( \text{ATLANTIS}_{20} \)	124.25	1,200	1,200	1,200	1,200	1,200
\( \text{ATLANTIS}_{30} \)	145.46	4,300	1,600	1,600	1,700	4,000
\( \text{ATLANTIS}_{40} \)	146.68	5,500	5,500	5,400	5,400	5,500
\( \text{ATLANTIS}_{50} \)	146.84	7,800	7,600	5,500	5,600	7,700
\( \text{ATLANTIS}_{70} \)	146.86	11,100	8,900	8,900	8,200	10,900
\( \text{BANKHEIST}_{10} \)	0.36	30	30	30	30	30
\( \text{BATTLEZONE}_{10} \)	115.32	2,000	2,000	2,000	2,000	2,000
\( \text{BEAMRIDER}_{20} \)	23.77	132	132	132	132	132
\( \text{BOWLING}_{30} \)	1.82	9	9	9	9	9
\( \text{BREAKOUT}_{10} \)	0.13	2	2	2	2	2
\( \text{BREAKOUT}_{20} \)	0.17	4	4	4	4	4
\( \text{BREAKOUT}_{30} \)	0.18	9	4	7	9	9
\( \text{BREAKOUT}_{40} \)	0.18	11	9	9	9	11
\( \text{BREAKOUT}_{50} \)	0.18	13	10	10	10	13
\( \text{BREAKOUT}_{70} \)	0.18	22	13	14	4	17
\( \text{BREAKOUT}_{100} \)	0.18	44	14	14	4	19
\( \text{BREAKOUT}_{200} \)	0.18	60	14	14	4	17
\( \text{CENTIPEDE}_{10} \)	141.37	1,923	922	1,632	1,511	1,621
\( \text{CHOPPERCOMMAND}_{10} \)	117.25	600	600	600	600	600
\( \text{CRAZYCLIMBER}_{20} \)	125	400	400	400	400	400
\( \text{CRAZYCLIMBER}_{30} \)	248.16	900	900	900	900	900
\( \text{DEMONATTACK}_{10} \)	8.31	50	50	50	50	50
\( \text{ENDURO}_{10} \)	0.04	6	6	6	4	6
\( \text{FISHINGDERBY}_{10} \)	0.16	8	6	8	6	8
\( \text{FREEWAY}_{10} \)	0.01	1	1	1	1	1
\( \text{FREEWAY}_{20} \)	0.02	2	2	2	2	2
\( \text{FREEWAY}_{30} \)	0.07	4	4	4	4	4
\( \text{FREEWAY}_{40} \)	0.1	5	5	5	5	5
\( \text{FREEWAY}_{50} \)	0.14	6	6	6	5	6
\( \text{FREEWAY}_{70} \)	0.23	9	9	9	7	9
\( \text{FREEWAY}_{100} \)	0.33	13	12	12	9	12
\( \text{FREEWAY}_{200} \)	0.72	25	24	25	12	20
\( \text{FROSTBITE}_{10} \)	9.13	70	70	70	70	70
\( \text{GOPHER}_{30} \)	1.43	20	20	20	20	20
\( \text{GOPHER}_{40} \)	10.05	180	100	100	100	180
\( \text{HERO}_{10} \)	8.18	75	75	75	75	75
\( \text{ICEHOCKEY}_{10} \)	0.01	1	1	1	1	1
\( \text{KANGAROO}_{20} \)	1.02	200	200	200	200	200
KANGAROO30	3.84	500	200	200	400	400
MONTEZUMAREVenge15	0	100	0	0	0	0
MSPACMAN20	148.91	480	460	470	470	470
NAMETHISGAME20	11.22	180	180	180	180	180
PHOENIX10	23.43	260	260	260	260	260
PONG20	-2.71	-1	-1	-1	-1	-1
PONG30	-4.25	-1	-1	-1	-1	-1
PONG40	-5.99	0	-1	0	0	0
PONG50	-7.85	1	-1	1	0	0
PONG70	-11.47	2	-1	0	-1	0
PONG100	-16.91	4	-1	-1	-1	1
PRIVATEEYE10	0.11	100	100	100	100	100
QBERT10	50.95	425	425	425	425	425
QBERT20	87.21	675	675	675	650	675
ROADRUNNER10	0.05	600	600	600	600	600
SEAQUEST10	0.78	20	20	20	20	20
SKIING10	-12,652.16	-7,066	-8,149	-7,078	-7,069	-7,066
SPACEINVADERS10	3.46	40	40	40	40	40
TENNIS10	-0.75	1	0	1	1	1
TIMEPILOT10	8.17	200	200	200	200	200
TUTANKHAM10	0.07	23	23	23	23	23
VIDEOPINBALL10	108.99	4,100	3,000	4,100	4,100	4,100
WIZARDOFWOR20	11.5	500	100	100	300	400
BIGFISH10	0.37	4	4	4	4	4
BIGFISH10	0.73	3	3	3	3	3
BIGFISH10	2.98	8	7	8	7	7
BIGFISH10	0.03	3	3	3	3	3
CHASER10	0.38	0.88	0.88	0.88	1	1
CHASER20	0.33	0.88	0.88	0.88	1	1
CHASER20	0.37	0.88	0.88	0.88	1	1
CHASER20	0.39	0.88	0.88	0.84	1	1
CLIMBER10	0.14	2	2	2	2	2
CLIMBER10	0.02	2	2	2	2	2
CLIMBER10	0.03	11	11	11	11	11
CLIMBER10	0.31	2	1	1	1	2
COINRUN10	0	10	10	10	10	10
COINRUN10	0	10	10	0	10	10
COINRUN10	0	10	10	0	10	10
COINRUN10	0.08	10	10	10	10	10
DODGEBALL10	0.08	16	16	16	16	16
DODGEBALL10	0.24	8	8	8	6	8
DODGEBALL10	0.1	8	8	8	8	8
DODGEBALL10	0.04	8	6	6	6	8
FRUITBOT40	-4.04	5	2	5	2	5
FRUITBOT40	-2.09	7	4	7	4	5
FRUITBOT40	-5.84	1	1	1	1	1
FRUITBOT40	-0.42	10	5	10	0	9
HEIST10	0	10	0	0	0	10
JUMPER10	0	10	0	0	0	0
JUMPER10	4.99	10	10	10	10	10
JUMPER20	4.96	10	10	10	10	10
JUMPER20	0.06	10	10	10	10	10
JUMPER20	0	10	0	0	0	0
LEAPER10	0	10	0	0	0	0
LEAPER20	0	10	10	10	10	10
LEAPER20	0	10	0	0	0	0
LEAPER20	0	10	0	0	0	0
MAZE30	6.1	10	10	10	10	10
MAZE30	0	10	0	0	0	0
MAZE30	5.65	10	10	10	10	10
MAZEEX100	0	10	0	0	0	0
MINERE10	0	1	1	1	1	1
MINERE10	0.05	2	2	2	2	2
MINERE2	0.11	1	1	1	1	1
MINERH0	0.04	3	3	3	3	3
NINJA E0	0	10	10	0	10	10
NINJA E1	0	10	10	0	10	10
NINJA E2	0	10	0	0	0	0
NINJA H0	0	10	0	0	0	0
PLUNDER E0	0.06	1	1	1	1	1
PLUNDER E1	0.17	1	1	1	1	1
PLUNDER E2	0	1	1	1	1	1
PLUNDER H0	0.07	1	1	1	1	1
STARPILOT E0	0.23	10	9	10	10	10
STARPILOT E1	0.14	5	5	5	5	5
STARPILOT E2	0.08	5	5	5	5	5
STARPILOT H0	0.17	5	4	5	4	5
EMPTY-5X5	0.76	1	1	1	1	1
EMPTY-6X6	0.5	1	1	1	1	1
EMPTY-8X8	0.18	1	1	1	1	1
EMPTY-16X16	0	1	1	0	0	1
DOORKEY-5X5	0.01	1	1	1	1	1
DOORKEY-6X6	0	1	1	1	0	1
DOORKEY-8X8	0	1	0	0	0	0
DOORKEY-16X16	0	1	0	0	0	0
MULTIROOM-N2-S4	0.09	1	1	1	1	1
MULTIROOM-N4-S5	0	1	0	0	0	0
MULTIROOM-N6	0	1	0	0	0	0
KEYCORRIDORS3R1	0	1	1	1	1	1
KEYCORRIDORS3R2	0	1	1	1	0	1
KEYCORRIDORS3R3	0	1	0	0	0	1
KEYCORRIDORS4R3	0	1	0	0	0	0
UNLOCK	0	1	1	1	1	1
UNLOCKPickUP	0	1	1	0	0	1
BLOCKEDUNLOCKPickUP	0	1	0	0	0	0
OBSTRUCTEDMAZE-1DL	0	1	1	0	0	1
OBSTRUCTEDMAZE-1DLH	0	1	0	0	0	1
OBSTRUCTEDMAZE-1DLHB	0	1	0	0	0	0
FOURROOMS	0	1	1	0	0	1
LAVACROSSINGS9N1	0	1	1	1	1	1
LAVACROSSINGS9N2	0	1	1	1	1	1
LAVACROSSINGS9N3	0	1	1	0	0	1
LAVACROSSINGS11N5	0	1	0	0	0	0
SIMPLECROSSINGS9N1	0.01	1	1	1	1	1
SIMPLECROSSINGS9N2	0	1	1	1	1	1
SIMPLECROSSINGS9N3	0	1	1	1	1	1
SIMPLECROSSINGS11N5	0	1	0	0	0	0
LAVAGAPS5	0.07	1	1	1	1	1
LAVAGAPS6	0.02	1	1	1	1	1
LAVAGAPS7	0.01	1	1	1	1	1