Repository: rlworkgroup/garage

Issue:
Change VecEnvExecutor to not set done to True in off policy algos
`garage.tf.envs.VecEnvExecutor` sets `done = True` when it reaches the `max_path_length`. This behavior blocks the off-policy algorithms to reach the highest performance. 


Discussion:

Resolving patch:
diff --git a/examples/tf/ddpg_pendulum.py b/examples/tf/ddpg_pendulum.py
--- a/examples/tf/ddpg_pendulum.py
+++ b/examples/tf/ddpg_pendulum.py
@@ -49,7 +49,6 @@ def run_task(*_):
             qf=qf,
             replay_buffer=replay_buffer,
             target_update_tau=1e-2,
-            max_path_length=100,
             n_train_steps=50,
             discount=0.9,
             min_buffer_size=int(1e4),
@@ -59,7 +58,7 @@ def run_task(*_):
 
         runner.setup(algo=ddpg, env=env)
 
-        runner.train(n_epochs=500, n_epoch_cycles=20)
+        runner.train(n_epochs=500, n_epoch_cycles=20, batch_size=100)
 
 
 run_experiment(
diff --git a/garage/tf/algos/off_policy_rl_algorithm.py b/garage/tf/algos/off_policy_rl_algorithm.py
--- a/garage/tf/algos/off_policy_rl_algorithm.py
+++ b/garage/tf/algos/off_policy_rl_algorithm.py
@@ -21,7 +21,7 @@ def __init__(
             discount=0.99,
             n_epochs=500,
             n_epoch_cycles=20,
-            max_path_length=100,
+            max_path_length=None,
             n_train_steps=50,
             buffer_batch_size=64,
             min_buffer_size=int(1e4),
diff --git a/garage/tf/samplers/off_policy_vectorized_sampler.py b/garage/tf/samplers/off_policy_vectorized_sampler.py
--- a/garage/tf/samplers/off_policy_vectorized_sampler.py
+++ b/garage/tf/samplers/off_policy_vectorized_sampler.py
@@ -57,11 +57,12 @@ def shutdown_worker(self):
         self.vec_env.close()
 
     @overrides
-    def obtain_samples(self, itr, batch_size=None):
+    def obtain_samples(self, itr, batch_size):
         """
         Collect samples for the given iteration number.
 
         :param itr: Iteration number.
+        :param batch_size: Batch size.
         :return: A list of paths.
         """
         paths = []
@@ -69,8 +70,6 @@ def obtain_samples(self, itr, batch_size=None):
         dones = np.asarray([True] * self.vec_env.num_envs)
         running_paths = [None] * self.vec_env.num_envs
         n_samples = 0
-        if not batch_size:
-            batch_size = self.n_envs * self.algo.max_path_length
 
         policy = self.algo.policy
         if self.algo.es:


Test patch:
diff --git a/tests/benchmarks/test_benchmark_ddpg.py b/tests/benchmarks/test_benchmark_ddpg.py
--- a/tests/benchmarks/test_benchmark_ddpg.py
+++ b/tests/benchmarks/test_benchmark_ddpg.py
@@ -161,7 +161,6 @@ def run_garage(env, seed, log_dir):
             policy_lr=params["policy_lr"],
             qf_lr=params["qf_lr"],
             target_update_tau=params["tau"],
-            max_path_length=params["n_rollout_steps"],
             n_train_steps=params["n_train_steps"],
             discount=params["discount"],
             min_buffer_size=int(1e4),
@@ -178,7 +177,8 @@ def run_garage(env, seed, log_dir):
         runner.setup(ddpg, env)
         runner.train(
             n_epochs=params['n_epochs'],
-            n_epoch_cycles=params['n_epoch_cycles'])
+            n_epoch_cycles=params['n_epoch_cycles'],
+            batch_size=params["n_rollout_steps"])
 
         garage_logger.remove_tabular_output(tabular_log_file)
 
diff --git a/tests/benchmarks/test_benchmark_her.py b/tests/benchmarks/test_benchmark_her.py
--- a/tests/benchmarks/test_benchmark_her.py
+++ b/tests/benchmarks/test_benchmark_her.py
@@ -150,7 +150,6 @@ def run_garage(env, seed, log_dir):
             target_update_tau=params["tau"],
             n_epochs=params["n_epochs"],
             n_epoch_cycles=params["n_epoch_cycles"],
-            max_path_length=params["n_rollout_steps"],
             n_train_steps=params["n_train_steps"],
             discount=params["discount"],
             exploration_strategy=action_noise,
@@ -168,7 +167,8 @@ def run_garage(env, seed, log_dir):
         runner.setup(algo, env)
         runner.train(
             n_epochs=params['n_epochs'],
-            n_epoch_cycles=params['n_epoch_cycles'])
+            n_epoch_cycles=params['n_epoch_cycles'],
+            batch_size=params["n_rollout_steps"])
 
         garage_logger.remove_tabular_output(tabular_log_file)
 
diff --git a/tests/garage/tf/algos/test_ddpg.py b/tests/garage/tf/algos/test_ddpg.py
--- a/tests/garage/tf/algos/test_ddpg.py
+++ b/tests/garage/tf/algos/test_ddpg.py
@@ -44,14 +44,14 @@ def test_ddpg_pendulum(self):
                 qf=qf,
                 replay_buffer=replay_buffer,
                 target_update_tau=1e-2,
-                max_path_length=100,
                 n_train_steps=50,
                 discount=0.9,
                 min_buffer_size=int(1e4),
                 exploration_strategy=action_noise,
             )
             runner.setup(algo, env)
-            last_avg_ret = runner.train(n_epochs=10, n_epoch_cycles=20)
+            last_avg_ret = runner.train(
+                n_epochs=10, n_epoch_cycles=20, batch_size=100)
             assert last_avg_ret > 60
 
             env.close()