Push model

README.md

@@ -51,6 +51,6 @@ uv run a2c.py
  'push_model': True,
  'seed': 1,
  'total_timesteps': 150_000,
- 'video_capture_frequency': 100}
+ 'video_capture_frequency': 50}
 ```
     

a2c.py

@@ -37,7 +37,7 @@ class HyperParams:
     """The number of parallel environments to run."""
     seed: int = 1
     """The random seed for reproducibility."""
-    video_capture_frequency: int = 100
+    video_capture_frequency: int = 50
     """The interval (in episodes) to record videos of the agent's performance."""
 
     total_timesteps: int = 150_000
@@ -125,12 +125,15 @@ class ActorCritic(nn.Module):
         logits = self.actor(states)
         return values, logits
 
-    def act(self, states: torch.Tensor):
+    def act(
+        self, states: torch.Tensor
+    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
         values, logits = self.forward(states)
         pd = torch.distributions.Categorical(logits=logits)
         actions = pd.sample()
         logprobs = pd.log_prob(actions)
-        return actions, logprobs, pd.entropy(), values
+        entropy = pd.entropy()
+        return actions, logprobs, entropy, values
 
 
 def main() -> None:
@@ -173,6 +176,7 @@ def main() -> None:
         values = torch.zeros(args.num_steps, envs.num_envs, device=device)
         rewards = torch.zeros(args.num_steps, envs.num_envs, device=device)
         logprobs = torch.zeros(args.num_steps, envs.num_envs, device=device)
+        entropies = torch.zeros(args.num_steps, envs.num_envs, device=device)
         masks = torch.zeros(args.num_steps, envs.num_envs, device=device)
 
         for t in range(args.num_steps):
@@ -196,6 +200,7 @@ def main() -> None:
             values[t] = value.squeeze()
             rewards[t] = torch.from_numpy(reward)
             logprobs[t] = logprob
+            entropies[t] = entropy
             masks[t] = torch.from_numpy(~terminations)
 
         advantages = torch.zeros_like(rewards).to(device)
@@ -207,7 +212,7 @@ def main() -> None:
 
         critic_loss = advantages.pow(2).mean()
         actor_loss = (
-            -(logprobs * advantages.detach()).mean() - args.ent_coef * entropy.mean()
+            -(logprobs * advantages.detach()).mean() - args.ent_coef * entropies.mean()
         )
 
         loss = actor_loss + critic_loss
@@ -219,7 +224,7 @@ def main() -> None:
         if step % args.log_interval < envs.num_envs:
             writer.add_scalar("losses/actor_loss", actor_loss, step)
             writer.add_scalar("losses/critic_loss", critic_loss, step)
-            writer.add_scalar("losses/entropy", entropy.mean(), step)
+            writer.add_scalar("losses/entropy", entropies.mean(), step)
             writer.add_scalar("charts/SPS", step // (time.time() - start_time), step)
             writer.add_scalar("losses/total_loss", loss, step)
             writer.add_scalar("losses/value_estimate", values.mean().item(), step)
@@ -228,7 +233,7 @@ def main() -> None:
                 actor_loss=actor_loss.item(),
                 critic_loss=critic_loss.item(),
                 # total_loss=loss.item(),
-                # entropy=entropy.mean().item(),
+                # entropy=entropies.mean().item(),
                 # value_estimate=values.mean().item(),
                 advantage=advantages.mean().item(),
                 sps=step // (time.time() - start_time),

events.out.tfevents.1757726955.Alperens-MBP.local.88268.0 → events.out.tfevents.1757936575.Alperens-MBP.local.22547.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d45664a03cebe436ec067c370681c75ddcd17dc635617f618d93432d88e283e5
-size 1928850
+oid sha256:150f0c68315d117958a752772a5d9b7569a9e9e03c3de4ffdab6ac2d2bc78bf0
+size 1928849

hyperparameters.json

@@ -3,7 +3,7 @@
   "exp_name": "a2c",
   "n_envs": 32,
   "seed": 1,
-  "video_capture_frequency": 100,
+  "video_capture_frequency": 50,
   "total_timesteps": 150000,
   "num_steps": 20,
   "gamma": 0.99,

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "hellrl"
 version = "0.1.0"
-description = "Add your description here"
+description = "RL Implementations with Cutting Edge Versions and Vectorized Training"
 readme = "README.md"
 authors = [
     { name = "Alperen ÜNLÜ"}