A2C playing MountainCar-v0 from https://github.com/sgoodfriend/rl-algo-impls/tree/0760ef7d52b17f30219a27c18ba52c8895025ae3

README.md

@@ -23,7 +23,7 @@ model-index:
 
 This is a trained model of a **A2C** agent playing **MountainCar-v0** using the [/sgoodfriend/rl-algo-impls](https://github.com/sgoodfriend/rl-algo-impls) repo.
 
-All models trained at this commit can be found at None.
+All models trained at this commit can be found at https://api.wandb.ai/links/sgoodfriend/eyvb72mv.
 
 ## Training Results
 
@@ -83,7 +83,7 @@ notebook.
 
 
 ## Benchmarking (with Lambda Labs instance)
-This and other models from None were generated by running a script on a Lambda
+This and other models from https://api.wandb.ai/links/sgoodfriend/eyvb72mv were generated by running a script on a Lambda
 Labs instance. In a Lambda Labs instance terminal:
 ```
 git clone git@github.com:sgoodfriend/rl-algo-impls.git

replay.meta.json

@@ -1 +1 @@
-{"content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 5.1.2 Copyright (c) 2000-2022 the FFmpeg developers\\nbuilt with clang version 14.0.6\\nconfiguration: --prefix=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pl --cc=arm64-apple-darwin20.0.0-clang --cxx=arm64-apple-darwin20.0.0-clang++ --nm=arm64-apple-darwin20.0.0-nm --ar=arm64-apple-darwin20.0.0-ar --disable-doc --disable-openssl --enable-demuxer=dash --enable-hardcoded-tables --enable-libfreetype --enable-libfontconfig --enable-libopenh264 --enable-cross-compile --arch=arm64 --target-os=darwin --cross-prefix=arm64-apple-darwin20.0.0- --host-cc=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_build_env/bin/x86_64-apple-darwin13.4.0-clang --enable-neon --enable-gnutls --enable-libmp3lame --enable-libvpx --enable-pthreads --enable-gpl --enable-libx264 --enable-libx265 --enable-libaom --enable-libsvtav1 --enable-libxml2 --enable-pic --enable-shared --disable-static --enable-version3 --enable-zlib --pkg-config=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_build_env/bin/pkg-config\\nlibavutil      57. 28.100 / 57. 28.100\\nlibavcodec     59. 37.100 / 59. 37.100\\nlibavformat    59. 27.100 / 59. 27.100\\nlibavdevice    59.  7.100 / 59.  7.100\\nlibavfilter     8. 44.100 /  8. 44.100\\nlibswscale      6.  7.100 /  6.  7.100\\nlibswresample   4.  7.100 /  4.  7.100\\nlibpostproc    56.  6.100 / 56.  6.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-f", "rawvideo", "-s:v", "1200x800", "-pix_fmt", "rgb24", "-framerate", "30", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-r", "30", "/var/folders/9g/my5557_91xddp6lx00nkzly80000gn/T/tmptrol4bye/a2c-MountainCar-v0/replay.mp4"]}, "episode": {"r": -118.0, "l": 118, "t": 42.409465}}
+{"content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 5.1.2 Copyright (c) 2000-2022 the FFmpeg developers\\nbuilt with clang version 14.0.6\\nconfiguration: --prefix=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pl --cc=arm64-apple-darwin20.0.0-clang --cxx=arm64-apple-darwin20.0.0-clang++ --nm=arm64-apple-darwin20.0.0-nm --ar=arm64-apple-darwin20.0.0-ar --disable-doc --disable-openssl --enable-demuxer=dash --enable-hardcoded-tables --enable-libfreetype --enable-libfontconfig --enable-libopenh264 --enable-cross-compile --arch=arm64 --target-os=darwin --cross-prefix=arm64-apple-darwin20.0.0- --host-cc=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_build_env/bin/x86_64-apple-darwin13.4.0-clang --enable-neon --enable-gnutls --enable-libmp3lame --enable-libvpx --enable-pthreads --enable-gpl --enable-libx264 --enable-libx265 --enable-libaom --enable-libsvtav1 --enable-libxml2 --enable-pic --enable-shared --disable-static --enable-version3 --enable-zlib --pkg-config=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_build_env/bin/pkg-config\\nlibavutil      57. 28.100 / 57. 28.100\\nlibavcodec     59. 37.100 / 59. 37.100\\nlibavformat    59. 27.100 / 59. 27.100\\nlibavdevice    59.  7.100 / 59.  7.100\\nlibavfilter     8. 44.100 /  8. 44.100\\nlibswscale      6.  7.100 /  6.  7.100\\nlibswresample   4.  7.100 /  4.  7.100\\nlibpostproc    56.  6.100 / 56.  6.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-f", "rawvideo", "-s:v", "1200x800", "-pix_fmt", "rgb24", "-framerate", "30", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-r", "30", "/var/folders/9g/my5557_91xddp6lx00nkzly80000gn/T/tmprlq4z8zm/a2c-MountainCar-v0/replay.mp4"]}, "episode": {"r": -118.0, "l": 118, "t": 1.678512}}

runner/env.py

@@ -26,6 +26,7 @@ from wrappers.initial_step_truncate_wrapper import InitialStepTruncateWrapper
 from wrappers.is_vector_env import IsVectorEnv
 from wrappers.noop_env_seed import NoopEnvSeed
 from wrappers.normalize import NormalizeObservation, NormalizeReward
+from wrappers.sync_vector_env_render_compat import SyncVectorEnvRenderCompat
 from wrappers.transpose_image_observation import TransposeImageObservation
 from wrappers.vectorable_wrapper import VecEnv
 from wrappers.video_compat_wrapper import VideoCompatWrapper
@@ -180,6 +181,8 @@ def _make_vec_env(
     else:
         raise ValueError(f"env_type {env_type} unsupported")
     envs = VecEnvClass([make(i) for i in range(n_envs)])
+    if env_type == "gymvec" and vec_env_class == "sync":
+        envs = SyncVectorEnvRenderCompat(envs)
     if training:
         assert tb_writer
         envs = EpisodeStatsWriter(

shared/callbacks/eval_callback.py

@@ -75,7 +75,9 @@ def evaluate(
     print_returns: bool = True,
     ignore_first_episode: bool = False,
 ) -> EpisodesStats:
+    policy.sync_normalization(env)
     policy.eval()
+
     episodes = EvaluateAccumulator(
         env.num_envs, n_episodes, print_returns, ignore_first_episode
     )
@@ -137,7 +139,6 @@ class EvalCallback(Callback):
     def on_step(self, timesteps_elapsed: int = 1) -> bool:
         super().on_step(timesteps_elapsed)
         if self.timesteps_elapsed // self.step_freq >= len(self.stats):
-            self.policy.sync_normalization(self.env)
             self.evaluate()
         return True
 
@@ -176,7 +177,6 @@ class EvalCallback(Callback):
             )
             if strictly_better and self.record_best_videos:
                 assert self.video_env and self.best_video_dir
-                self.policy.sync_normalization(self.video_env)
                 self.best_video_base_path = os.path.join(
                     self.best_video_dir, str(self.timesteps_elapsed)
                 )

wrappers/sync_vector_env_render_compat.py

@@ -0,0 +1,31 @@
+import numpy as np
+
+from gym.vector.sync_vector_env import SyncVectorEnv
+from stable_baselines3.common.vec_env.base_vec_env import tile_images
+from typing import Optional
+
+from wrappers.vectorable_wrapper import (
+    VecotarableWrapper,
+)
+
+
+class SyncVectorEnvRenderCompat(VecotarableWrapper):
+    def __init__(self, env) -> None:
+        super().__init__(env)
+
+    def render(self, mode: str = "human") -> Optional[np.ndarray]:
+        base_env = self.env.unwrapped
+        if isinstance(base_env, SyncVectorEnv):
+            imgs = [env.render(mode="rgb_array") for env in base_env.envs]
+            bigimg = tile_images(imgs)
+            if mode == "human":
+                import cv2
+
+                cv2.imshow("vecenv", bigimg[:, :, ::-1])
+                cv2.waitKey(1)
+            elif mode == "rgb_array":
+                return bigimg
+            else:
+                raise NotImplemented(f"Render mode {mode} is not supported")
+        else:
+            return self.env.render(mode=mode)

wrappers/vec_episode_recorder.py

@@ -1,9 +1,6 @@
 import numpy as np
 
-from gym.vector.sync_vector_env import SyncVectorEnv
 from gym.wrappers.monitoring.video_recorder import VideoRecorder
-from stable_baselines3.common.vec_env.base_vec_env import tile_images
-from typing import Optional
 
 from wrappers.vectorable_wrapper import (
     VecotarableWrapper,
@@ -45,7 +42,7 @@ class VecEpisodeRecorder(VecotarableWrapper):
         self._close_video_recorder()
 
         self.video_recorder = VideoRecorder(
-            SyncVectorEnvRenderCompat(self.env),
+            self.env,
             base_path=self.base_path,
         )
 
@@ -56,25 +53,3 @@ class VecEpisodeRecorder(VecotarableWrapper):
         if self.video_recorder:
             self.video_recorder.close()
         self.video_recorder = None
-
-
-class SyncVectorEnvRenderCompat(VecotarableWrapper):
-    def __init__(self, env) -> None:
-        super().__init__(env)
-
-    def render(self, mode: str = "human") -> Optional[np.ndarray]:
-        base_env = self.env.unwrapped
-        if isinstance(base_env, SyncVectorEnv):
-            imgs = [env.render(mode="rgb_array") for env in base_env.envs]
-            bigimg = tile_images(imgs)
-            if mode == "humnan":
-                import cv2
-
-                cv2.imshow("vecenv", bigimg[:, :, ::-1])
-                cv2.waitKey(1)
-            elif mode == "rgb_array":
-                return bigimg
-            else:
-                raise NotImplemented(f"Render mode {mode} is not supported")
-        else:
-            return self.env.render(mode=mode)