A2C playing LunarLander-v2 from https://github.com/sgoodfriend/rl-algo-impls/tree/0760ef7d52b17f30219a27c18ba52c8895025ae3

README.md

@@ -10,7 +10,7 @@ model-index:
   results:
   - metrics:
     - type: mean_reward
-      value: 203.36 +/- 113.36
+      value: 203.36 +/- 113.35
       name: mean_reward
     task:
       type: reinforcement-learning
@@ -31,9 +31,9 @@ This model was trained from 3 trainings of **A2C** agents using different initia
 
 | algo   | env            |   seed |   reward_mean |   reward_std |   eval_episodes | best   | wandb_url                                                                    |
 |:-------|:---------------|-------:|--------------:|-------------:|----------------:|:-------|:-----------------------------------------------------------------------------|
-| a2c    | LunarLander-v2 |      1 |      116.479  |      101.181 |              16 |        | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/wzm9jhee) |
-| a2c    | LunarLander-v2 |      2 |       74.1282 |      134.996 |              16 |        | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/f6jaeow9) |
-| a2c    | LunarLander-v2 |      3 |      203.36   |      113.362 |              16 | *      | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/nyywc8ns) |
+| a2c    | LunarLander-v2 |      1 |      102.539  |      106.633 |              16 |        | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/wzm9jhee) |
+| a2c    | LunarLander-v2 |      2 |       86.8229 |      132.518 |              16 |        | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/f6jaeow9) |
+| a2c    | LunarLander-v2 |      3 |      203.362  |      113.355 |              16 | *      | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/nyywc8ns) |
 
 
 ### Prerequisites: Weights & Biases (WandB)

replay.meta.json

@@ -1 +1 @@
-{"content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 4.2.7-0ubuntu0.1 Copyright (c) 2000-2022 the FFmpeg developers\\nbuilt with gcc 9 (Ubuntu 9.4.0-1ubuntu1~20.04.1)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --arch=amd64 --enable-gpl --disable-stripping --enable-avresample --disable-filter=resample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librsvg --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-nvenc --enable-chromaprint --enable-frei0r --enable-libx264 --enable-shared\\nlibavutil      56. 31.100 / 56. 31.100\\nlibavcodec     58. 54.100 / 58. 54.100\\nlibavformat    58. 29.100 / 58. 29.100\\nlibavdevice    58.  8.100 / 58.  8.100\\nlibavfilter     7. 57.100 /  7. 57.100\\nlibavresample   4.  0.  0 /  4.  0.  0\\nlibswscale      5.  5.100 /  5.  5.100\\nlibswresample   3.  5.100 /  3.  5.100\\nlibpostproc    55.  5.100 / 55.  5.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-f", "rawvideo", "-s:v", "600x400", "-pix_fmt", "rgb24", "-framerate", "50", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-r", "50", "/tmp/tmpmog2i1xe/a2c-LunarLander-v2/replay.mp4"]}, "episode": {"r": 3.8731765747070312, "l": 123, "t": 1.6477}}
+{"content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 5.1.2 Copyright (c) 2000-2022 the FFmpeg developers\\nbuilt with clang version 14.0.6\\nconfiguration: --prefix=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pl --cc=arm64-apple-darwin20.0.0-clang --cxx=arm64-apple-darwin20.0.0-clang++ --nm=arm64-apple-darwin20.0.0-nm --ar=arm64-apple-darwin20.0.0-ar --disable-doc --disable-openssl --enable-demuxer=dash --enable-hardcoded-tables --enable-libfreetype --enable-libfontconfig --enable-libopenh264 --enable-cross-compile --arch=arm64 --target-os=darwin --cross-prefix=arm64-apple-darwin20.0.0- --host-cc=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_build_env/bin/x86_64-apple-darwin13.4.0-clang --enable-neon --enable-gnutls --enable-libmp3lame --enable-libvpx --enable-pthreads --enable-gpl --enable-libx264 --enable-libx265 --enable-libaom --enable-libsvtav1 --enable-libxml2 --enable-pic --enable-shared --disable-static --enable-version3 --enable-zlib --pkg-config=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_build_env/bin/pkg-config\\nlibavutil      57. 28.100 / 57. 28.100\\nlibavcodec     59. 37.100 / 59. 37.100\\nlibavformat    59. 27.100 / 59. 27.100\\nlibavdevice    59.  7.100 / 59.  7.100\\nlibavfilter     8. 44.100 /  8. 44.100\\nlibswscale      6.  7.100 /  6.  7.100\\nlibswresample   4.  7.100 /  4.  7.100\\nlibpostproc    56.  6.100 / 56.  6.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-f", "rawvideo", "-s:v", "1200x800", "-pix_fmt", "rgb24", "-framerate", "50", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-r", "50", "/var/folders/9g/my5557_91xddp6lx00nkzly80000gn/T/tmps7cn24ni/a2c-LunarLander-v2/replay.mp4"]}, "episode": {"r": 31.878372192382812, "l": 302, "t": 3.747}}

runner/env.py

@@ -26,6 +26,7 @@ from wrappers.initial_step_truncate_wrapper import InitialStepTruncateWrapper
 from wrappers.is_vector_env import IsVectorEnv
 from wrappers.noop_env_seed import NoopEnvSeed
 from wrappers.normalize import NormalizeObservation, NormalizeReward
+from wrappers.sync_vector_env_render_compat import SyncVectorEnvRenderCompat
 from wrappers.transpose_image_observation import TransposeImageObservation
 from wrappers.vectorable_wrapper import VecEnv
 from wrappers.video_compat_wrapper import VideoCompatWrapper
@@ -180,6 +181,8 @@ def _make_vec_env(
     else:
         raise ValueError(f"env_type {env_type} unsupported")
     envs = VecEnvClass([make(i) for i in range(n_envs)])
+    if env_type == "gymvec" and vec_env_class == "sync":
+        envs = SyncVectorEnvRenderCompat(envs)
     if training:
         assert tb_writer
         envs = EpisodeStatsWriter(

shared/callbacks/eval_callback.py

@@ -75,7 +75,9 @@ def evaluate(
     print_returns: bool = True,
     ignore_first_episode: bool = False,
 ) -> EpisodesStats:
+    policy.sync_normalization(env)
     policy.eval()
+
     episodes = EvaluateAccumulator(
         env.num_envs, n_episodes, print_returns, ignore_first_episode
     )
@@ -137,7 +139,6 @@ class EvalCallback(Callback):
     def on_step(self, timesteps_elapsed: int = 1) -> bool:
         super().on_step(timesteps_elapsed)
         if self.timesteps_elapsed // self.step_freq >= len(self.stats):
-            self.policy.sync_normalization(self.env)
             self.evaluate()
         return True
 
@@ -176,7 +177,6 @@ class EvalCallback(Callback):
             )
             if strictly_better and self.record_best_videos:
                 assert self.video_env and self.best_video_dir
-                self.policy.sync_normalization(self.video_env)
                 self.best_video_base_path = os.path.join(
                     self.best_video_dir, str(self.timesteps_elapsed)
                 )

wrappers/sync_vector_env_render_compat.py

@@ -0,0 +1,31 @@
+import numpy as np
+
+from gym.vector.sync_vector_env import SyncVectorEnv
+from stable_baselines3.common.vec_env.base_vec_env import tile_images
+from typing import Optional
+
+from wrappers.vectorable_wrapper import (
+    VecotarableWrapper,
+)
+
+
+class SyncVectorEnvRenderCompat(VecotarableWrapper):
+    def __init__(self, env) -> None:
+        super().__init__(env)
+
+    def render(self, mode: str = "human") -> Optional[np.ndarray]:
+        base_env = self.env.unwrapped
+        if isinstance(base_env, SyncVectorEnv):
+            imgs = [env.render(mode="rgb_array") for env in base_env.envs]
+            bigimg = tile_images(imgs)
+            if mode == "human":
+                import cv2
+
+                cv2.imshow("vecenv", bigimg[:, :, ::-1])
+                cv2.waitKey(1)
+            elif mode == "rgb_array":
+                return bigimg
+            else:
+                raise NotImplemented(f"Render mode {mode} is not supported")
+        else:
+            return self.env.render(mode=mode)

wrappers/vec_episode_recorder.py

@@ -1,9 +1,6 @@
 import numpy as np
 
-from gym.vector.sync_vector_env import SyncVectorEnv
 from gym.wrappers.monitoring.video_recorder import VideoRecorder
-from stable_baselines3.common.vec_env.base_vec_env import tile_images
-from typing import Optional
 
 from wrappers.vectorable_wrapper import (
     VecotarableWrapper,
@@ -45,7 +42,7 @@ class VecEpisodeRecorder(VecotarableWrapper):
         self._close_video_recorder()
 
         self.video_recorder = VideoRecorder(
-            SyncVectorEnvRenderCompat(self.env),
+            self.env,
             base_path=self.base_path,
         )
 
@@ -56,25 +53,3 @@ class VecEpisodeRecorder(VecotarableWrapper):
         if self.video_recorder:
             self.video_recorder.close()
         self.video_recorder = None
-
-
-class SyncVectorEnvRenderCompat(VecotarableWrapper):
-    def __init__(self, env) -> None:
-        super().__init__(env)
-
-    def render(self, mode: str = "human") -> Optional[np.ndarray]:
-        base_env = self.env.unwrapped
-        if isinstance(base_env, SyncVectorEnv):
-            imgs = [env.render(mode="rgb_array") for env in base_env.envs]
-            bigimg = tile_images(imgs)
-            if mode == "humnan":
-                import cv2
-
-                cv2.imshow("vecenv", bigimg[:, :, ::-1])
-                cv2.waitKey(1)
-            elif mode == "rgb_array":
-                return bigimg
-            else:
-                raise NotImplemented(f"Render mode {mode} is not supported")
-        else:
-            return self.env.render(mode=mode)