Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+git.diff filter=lfs diff=lfs merge=lfs -text
+replay.mp4 filter=lfs diff=lfs merge=lfs -text

.summary/0/events.out.tfevents.1695669315.rhmmedcatt-proliant-ml350-gen10

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:559efd05d5ef461af40f23b9ac02bdc8c3cf9684cc1dedb99011ffb298672e94
+size 62072

.summary/1/events.out.tfevents.1695669315.rhmmedcatt-proliant-ml350-gen10

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5d552d7b5ffa9b296493ba3d5bb385c83c119daff9e205ba3d3dd3110b3997c
+size 54596

README.md

@@ -0,0 +1,56 @@
+---
+library_name: sample-factory
+tags:
+- deep-reinforcement-learning
+- reinforcement-learning
+- sample-factory
+model-index:
+- name: APPO
+  results:
+  - task:
+      type: reinforcement-learning
+      name: reinforcement-learning
+    dataset:
+      name: atari_beamrider
+      type: atari_beamrider
+    metrics:
+    - type: mean_reward
+      value: 550.00 +/- 86.33
+      name: mean_reward
+      verified: false
+---
+
+A(n) **APPO** model trained on the **atari_beamrider** environment.
+
+This model was trained using Sample-Factory 2.0: https://github.com/alex-petrenko/sample-factory.
+Documentation for how to use Sample-Factory can be found at https://www.samplefactory.dev/
+
+
+## Downloading the model
+
+After installing Sample-Factory, download the model with:
+```
+python -m sample_factory.huggingface.load_from_hub -r MattStammers/appo-atari_beamrider
+```
+
+    
+## Using the model
+
+To run the model after download, use the `enjoy` script corresponding to this environment:
+```
+python -m sf_examples.atari.enjoy_atari --algo=APPO --env=atari_beamrider --train_dir=./train_dir --experiment=appo-atari_beamrider
+```
+
+
+You can also upload models to the Hugging Face Hub using the same script with the `--push_to_hub` flag.
+See https://www.samplefactory.dev/10-huggingface/huggingface/ for more details
+    
+## Training with this model
+
+To continue training with this model, use the `train` script corresponding to this environment:
+```
+python -m sf_examples.atari.train_atari --algo=APPO --env=atari_beamrider --train_dir=./train_dir --experiment=appo-atari_beamrider --restart_behavior=resume --train_for_env_steps=10000000000
+```
+
+Note, you may have to adjust `--train_for_env_steps` to a suitably high number as the experiment will resume at the number of steps it concluded at.
+    

checkpoint_p0/best_000000752_192512_reward_2.966.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d27fa4b43bf1c4bb05ef203e58f236e03723942eb3e5af1de903f011dd357ad
+size 20740275

checkpoint_p0/checkpoint_000000880_225280.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af474c9a6ae1cb180abfafec10598af3d5e837471096d731ba55993c2e92e606
+size 20740611

checkpoint_p1/best_000000496_126976_reward_3.672.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2eaff53d62ad198ba9f2c7a83f3a5f0d95d018577ccaeed72ffa3aa6ddc2f9e0
+size 20740275

checkpoint_p1/checkpoint_000000880_225280.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5c017997b3ed2b4f105383cb39252ea612d3d8cfdb08b5e3792571adb0c2e3c
+size 20740611

config.json

@@ -0,0 +1,144 @@
+{
+  "help": false,
+  "algo": "APPO",
+  "env": "atari_beamrider",
+  "experiment": "atari_beamrider",
+  "train_dir": "./train_atari",
+  "restart_behavior": "resume",
+  "device": "gpu",
+  "seed": null,
+  "num_policies": 2,
+  "async_rl": false,
+  "serial_mode": false,
+  "batched_sampling": false,
+  "num_batches_to_accumulate": 2,
+  "worker_num_splits": 1,
+  "policy_workers_per_policy": 1,
+  "max_policy_lag": 1000,
+  "num_workers": 8,
+  "num_envs_per_worker": 1,
+  "batch_size": 256,
+  "num_batches_per_epoch": 4,
+  "num_epochs": 4,
+  "rollout": 128,
+  "recurrence": 1,
+  "shuffle_minibatches": false,
+  "gamma": 0.99,
+  "reward_scale": 1.0,
+  "reward_clip": 1000.0,
+  "value_bootstrap": false,
+  "normalize_returns": true,
+  "exploration_loss_coeff": 0.01,
+  "value_loss_coeff": 0.5,
+  "kl_loss_coeff": 0.0,
+  "exploration_loss": "entropy",
+  "gae_lambda": 0.95,
+  "ppo_clip_ratio": 0.1,
+  "ppo_clip_value": 1.0,
+  "with_vtrace": false,
+  "vtrace_rho": 1.0,
+  "vtrace_c": 1.0,
+  "optimizer": "adam",
+  "adam_eps": 1e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "max_grad_norm": 0.5,
+  "learning_rate": 0.00025,
+  "lr_schedule": "linear_decay",
+  "lr_schedule_kl_threshold": 0.008,
+  "lr_adaptive_min": 1e-06,
+  "lr_adaptive_max": 0.01,
+  "obs_subtract_mean": 0.0,
+  "obs_scale": 255.0,
+  "normalize_input": true,
+  "normalize_input_keys": null,
+  "decorrelate_experience_max_seconds": 0,
+  "decorrelate_envs_on_one_worker": true,
+  "actor_worker_gpus": [],
+  "set_workers_cpu_affinity": true,
+  "force_envs_single_thread": false,
+  "default_niceness": 0,
+  "log_to_file": true,
+  "experiment_summaries_interval": 3,
+  "flush_summaries_interval": 30,
+  "stats_avg": 100,
+  "summaries_use_frameskip": true,
+  "heartbeat_interval": 20,
+  "heartbeat_reporting_interval": 180,
+  "train_for_env_steps": 10000000,
+  "train_for_seconds": 10000000000,
+  "save_every_sec": 120,
+  "keep_checkpoints": 2,
+  "load_checkpoint_kind": "latest",
+  "save_milestones_sec": -1,
+  "save_best_every_sec": 5,
+  "save_best_metric": "reward",
+  "save_best_after": 100000,
+  "benchmark": false,
+  "encoder_mlp_layers": [
+    512,
+    512
+  ],
+  "encoder_conv_architecture": "convnet_atari",
+  "encoder_conv_mlp_layers": [
+    512
+  ],
+  "use_rnn": false,
+  "rnn_size": 512,
+  "rnn_type": "gru",
+  "rnn_num_layers": 1,
+  "decoder_mlp_layers": [],
+  "nonlinearity": "relu",
+  "policy_initialization": "orthogonal",
+  "policy_init_gain": 1.0,
+  "actor_critic_share_weights": true,
+  "adaptive_stddev": false,
+  "continuous_tanh_scale": 0.0,
+  "initial_stddev": 1.0,
+  "use_env_info_cache": false,
+  "env_gpu_actions": false,
+  "env_gpu_observations": true,
+  "env_frameskip": 4,
+  "env_framestack": 4,
+  "pixel_format": "CHW",
+  "use_record_episode_statistics": true,
+  "with_wandb": true,
+  "wandb_user": "matt-stammers",
+  "wandb_project": "atari",
+  "wandb_group": "atari_beamrider",
+  "wandb_job_type": "SF",
+  "wandb_tags": [
+    "atari"
+  ],
+  "with_pbt": false,
+  "pbt_mix_policies_in_one_env": true,
+  "pbt_period_env_steps": 5000000,
+  "pbt_start_mutation": 20000000,
+  "pbt_replace_fraction": 0.3,
+  "pbt_mutation_rate": 0.15,
+  "pbt_replace_reward_gap": 0.1,
+  "pbt_replace_reward_gap_absolute": 1e-06,
+  "pbt_optimize_gamma": false,
+  "pbt_target_objective": "true_objective",
+  "pbt_perturb_min": 1.1,
+  "pbt_perturb_max": 1.5,
+  "command_line": "--algo=APPO --env=atari_beamrider --experiment=atari_beamrider --num_policies=2 --train_dir=./train_atari --with_wandb=true --wandb_user=matt-stammers --wandb_project=atari --wandb_group=atari_beamrider --wandb_job_type=SF --wandb_tags=atari",
+  "cli_args": {
+    "algo": "APPO",
+    "env": "atari_beamrider",
+    "experiment": "atari_beamrider",
+    "train_dir": "./train_atari",
+    "num_policies": 2,
+    "with_wandb": true,
+    "wandb_user": "matt-stammers",
+    "wandb_project": "atari",
+    "wandb_group": "atari_beamrider",
+    "wandb_job_type": "SF",
+    "wandb_tags": [
+      "atari"
+    ]
+  },
+  "git_hash": "5fff97c2f535da5987d358cdbe6927cccd43621e",
+  "git_repo_name": "not a git repository",
+  "wandb_unique_id": "atari_beamrider_20230925_201508_276248"
+}

git.diff

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28f3fe545f35e5db1e419bf1e198c0bc5ef89459d55f59e2d06d3c108907e732
+size 14403396

replay.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf78ebefce98d2ed24463f365e952edb24143855d16a515555682269cdc9365b
+size 8980259

sf_log.txt

@@ -0,0 +1,274 @@
+[2023-09-25 20:15:19,439][95689] Saving configuration to ./train_atari/atari_beamrider/config.json...
+[2023-09-25 20:15:19,756][95689] Rollout worker 0 uses device cpu
+[2023-09-25 20:15:19,757][95689] Rollout worker 1 uses device cpu
+[2023-09-25 20:15:19,757][95689] Rollout worker 2 uses device cpu
+[2023-09-25 20:15:19,758][95689] Rollout worker 3 uses device cpu
+[2023-09-25 20:15:19,758][95689] Rollout worker 4 uses device cpu
+[2023-09-25 20:15:19,759][95689] Rollout worker 5 uses device cpu
+[2023-09-25 20:15:19,759][95689] Rollout worker 6 uses device cpu
+[2023-09-25 20:15:19,760][95689] Rollout worker 7 uses device cpu
+[2023-09-25 20:15:19,760][95689] In synchronous mode, we only accumulate one batch. Setting num_batches_to_accumulate to 1
+[2023-09-25 20:15:19,806][95689] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-09-25 20:15:19,806][95689] InferenceWorker_p0-w0: min num requests: 1
+[2023-09-25 20:15:19,810][95689] Using GPUs [1] for process 1 (actually maps to GPUs [1])
+[2023-09-25 20:15:19,810][95689] InferenceWorker_p1-w0: min num requests: 1
+[2023-09-25 20:15:19,834][95689] Starting all processes...
+[2023-09-25 20:15:19,834][95689] Starting process learner_proc0
+[2023-09-25 20:15:21,428][95689] Starting process learner_proc1
+[2023-09-25 20:15:21,431][96647] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-09-25 20:15:21,431][96647] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for learning process 0
+[2023-09-25 20:15:21,449][96647] Num visible devices: 1
+[2023-09-25 20:15:21,465][96647] Starting seed is not provided
+[2023-09-25 20:15:21,465][96647] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-09-25 20:15:21,465][96647] Initializing actor-critic model on device cuda:0
+[2023-09-25 20:15:21,466][96647] RunningMeanStd input shape: (4, 84, 84)
+[2023-09-25 20:15:21,466][96647] RunningMeanStd input shape: (1,)
+[2023-09-25 20:15:21,477][96647] ConvEncoder: input_channels=4
+[2023-09-25 20:15:21,635][96647] Conv encoder output size: 512
+[2023-09-25 20:15:21,637][96647] Created Actor Critic model with architecture:
+[2023-09-25 20:15:21,637][96647] ActorCriticSharedWeights(
+  (obs_normalizer): ObservationNormalizer(
+    (running_mean_std): RunningMeanStdDictInPlace(
+      (running_mean_std): ModuleDict(
+        (obs): RunningMeanStdInPlace()
+      )
+    )
+  )
+  (returns_normalizer): RecursiveScriptModule(original_name=RunningMeanStdInPlace)
+  (encoder): MultiInputEncoder(
+    (encoders): ModuleDict(
+      (obs): ConvEncoder(
+        (enc): RecursiveScriptModule(
+          original_name=ConvEncoderImpl
+          (conv_head): RecursiveScriptModule(
+            original_name=Sequential
+            (0): RecursiveScriptModule(original_name=Conv2d)
+            (1): RecursiveScriptModule(original_name=ReLU)
+            (2): RecursiveScriptModule(original_name=Conv2d)
+            (3): RecursiveScriptModule(original_name=ReLU)
+            (4): RecursiveScriptModule(original_name=Conv2d)
+            (5): RecursiveScriptModule(original_name=ReLU)
+          )
+          (mlp_layers): RecursiveScriptModule(
+            original_name=Sequential
+            (0): RecursiveScriptModule(original_name=Linear)
+            (1): RecursiveScriptModule(original_name=ReLU)
+          )
+        )
+      )
+    )
+  )
+  (core): ModelCoreIdentity()
+  (decoder): MlpDecoder(
+    (mlp): Identity()
+  )
+  (critic_linear): Linear(in_features=512, out_features=1, bias=True)
+  (action_parameterization): ActionParameterizationDefault(
+    (distribution_linear): Linear(in_features=512, out_features=9, bias=True)
+  )
+)
+[2023-09-25 20:15:22,224][96647] Using optimizer <class 'torch.optim.adam.Adam'>
+[2023-09-25 20:15:22,225][96647] No checkpoints found
+[2023-09-25 20:15:22,225][96647] Did not load from checkpoint, starting from scratch!
+[2023-09-25 20:15:22,225][96647] Initialized policy 0 weights for model version 0
+[2023-09-25 20:15:22,226][96647] LearnerWorker_p0 finished initialization!
+[2023-09-25 20:15:22,227][96647] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-09-25 20:15:23,053][95689] Starting all processes...
+[2023-09-25 20:15:23,057][96710] Using GPUs [1] for process 1 (actually maps to GPUs [1])
+[2023-09-25 20:15:23,057][96710] Set environment var CUDA_VISIBLE_DEVICES to '1' (GPU indices [1]) for learning process 1
+[2023-09-25 20:15:23,061][95689] Starting process inference_proc0-0
+[2023-09-25 20:15:23,061][95689] Starting process inference_proc1-0
+[2023-09-25 20:15:23,061][95689] Starting process rollout_proc0
+[2023-09-25 20:15:23,075][96710] Num visible devices: 1
+[2023-09-25 20:15:23,061][95689] Starting process rollout_proc1
+[2023-09-25 20:15:23,062][95689] Starting process rollout_proc2
+[2023-09-25 20:15:23,092][96710] Starting seed is not provided
+[2023-09-25 20:15:23,092][96710] Using GPUs [0] for process 1 (actually maps to GPUs [1])
+[2023-09-25 20:15:23,092][96710] Initializing actor-critic model on device cuda:0
+[2023-09-25 20:15:23,093][96710] RunningMeanStd input shape: (4, 84, 84)
+[2023-09-25 20:15:23,062][95689] Starting process rollout_proc3
+[2023-09-25 20:15:23,093][96710] RunningMeanStd input shape: (1,)
+[2023-09-25 20:15:23,063][95689] Starting process rollout_proc4
+[2023-09-25 20:15:23,066][95689] Starting process rollout_proc5
+[2023-09-25 20:15:23,067][95689] Starting process rollout_proc6
+[2023-09-25 20:15:23,067][95689] Starting process rollout_proc7
+[2023-09-25 20:15:23,105][96710] ConvEncoder: input_channels=4
+[2023-09-25 20:15:23,358][96710] Conv encoder output size: 512
+[2023-09-25 20:15:23,360][96710] Created Actor Critic model with architecture:
+[2023-09-25 20:15:23,361][96710] ActorCriticSharedWeights(
+  (obs_normalizer): ObservationNormalizer(
+    (running_mean_std): RunningMeanStdDictInPlace(
+      (running_mean_std): ModuleDict(
+        (obs): RunningMeanStdInPlace()
+      )
+    )
+  )
+  (returns_normalizer): RecursiveScriptModule(original_name=RunningMeanStdInPlace)
+  (encoder): MultiInputEncoder(
+    (encoders): ModuleDict(
+      (obs): ConvEncoder(
+        (enc): RecursiveScriptModule(
+          original_name=ConvEncoderImpl
+          (conv_head): RecursiveScriptModule(
+            original_name=Sequential
+            (0): RecursiveScriptModule(original_name=Conv2d)
+            (1): RecursiveScriptModule(original_name=ReLU)
+            (2): RecursiveScriptModule(original_name=Conv2d)
+            (3): RecursiveScriptModule(original_name=ReLU)
+            (4): RecursiveScriptModule(original_name=Conv2d)
+            (5): RecursiveScriptModule(original_name=ReLU)
+          )
+          (mlp_layers): RecursiveScriptModule(
+            original_name=Sequential
+            (0): RecursiveScriptModule(original_name=Linear)
+            (1): RecursiveScriptModule(original_name=ReLU)
+          )
+        )
+      )
+    )
+  )
+  (core): ModelCoreIdentity()
+  (decoder): MlpDecoder(
+    (mlp): Identity()
+  )
+  (critic_linear): Linear(in_features=512, out_features=1, bias=True)
+  (action_parameterization): ActionParameterizationDefault(
+    (distribution_linear): Linear(in_features=512, out_features=9, bias=True)
+  )
+)
+[2023-09-25 20:15:23,963][96710] Using optimizer <class 'torch.optim.adam.Adam'>
+[2023-09-25 20:15:23,964][96710] No checkpoints found
+[2023-09-25 20:15:23,964][96710] Did not load from checkpoint, starting from scratch!
+[2023-09-25 20:15:23,964][96710] Initialized policy 1 weights for model version 0
+[2023-09-25 20:15:23,966][96710] LearnerWorker_p1 finished initialization!
+[2023-09-25 20:15:23,966][96710] Using GPUs [0] for process 1 (actually maps to GPUs [1])
+[2023-09-25 20:15:24,995][96848] Using GPUs [0] for process 0 (actually maps to GPUs [0])
+[2023-09-25 20:15:24,995][96848] Set environment var CUDA_VISIBLE_DEVICES to '0' (GPU indices [0]) for inference process 0
+[2023-09-25 20:15:25,013][96848] Num visible devices: 1
+[2023-09-25 20:15:25,039][96885] Worker 3 uses CPU cores [12, 13, 14, 15]
+[2023-09-25 20:15:25,063][96887] Worker 5 uses CPU cores [20, 21, 22, 23]
+[2023-09-25 20:15:25,085][96882] Worker 1 uses CPU cores [4, 5, 6, 7]
+[2023-09-25 20:15:25,089][96849] Using GPUs [1] for process 1 (actually maps to GPUs [1])
+[2023-09-25 20:15:25,089][96849] Set environment var CUDA_VISIBLE_DEVICES to '1' (GPU indices [1]) for inference process 1
+[2023-09-25 20:15:25,109][96849] Num visible devices: 1
+[2023-09-25 20:15:25,110][96886] Worker 4 uses CPU cores [16, 17, 18, 19]
+[2023-09-25 20:15:25,130][96888] Worker 6 uses CPU cores [24, 25, 26, 27]
+[2023-09-25 20:15:25,160][96889] Worker 7 uses CPU cores [28, 29, 30, 31]
+[2023-09-25 20:15:25,210][96868] Worker 0 uses CPU cores [0, 1, 2, 3]
+[2023-09-25 20:15:25,262][96884] Worker 2 uses CPU cores [8, 9, 10, 11]
+[2023-09-25 20:15:25,632][96848] RunningMeanStd input shape: (4, 84, 84)
+[2023-09-25 20:15:25,632][96848] RunningMeanStd input shape: (1,)
+[2023-09-25 20:15:25,642][96848] ConvEncoder: input_channels=4
+[2023-09-25 20:15:25,710][95689] Fps is (10 sec: nan, 60 sec: nan, 300 sec: nan). Total num frames: 0. Throughput: 0: nan, 1: nan. Samples: 0. Policy #0 lag: (min: -1.0, avg: -1.0, max: -1.0)
+[2023-09-25 20:15:25,727][96849] RunningMeanStd input shape: (4, 84, 84)
+[2023-09-25 20:15:25,728][96849] RunningMeanStd input shape: (1,)
+[2023-09-25 20:15:25,739][96849] ConvEncoder: input_channels=4
+[2023-09-25 20:15:25,742][96848] Conv encoder output size: 512
+[2023-09-25 20:15:25,747][95689] Inference worker 0-0 is ready!
+[2023-09-25 20:15:25,838][96849] Conv encoder output size: 512
+[2023-09-25 20:15:25,843][95689] Inference worker 1-0 is ready!
+[2023-09-25 20:15:25,844][95689] All inference workers are ready! Signal rollout workers to start!
+[2023-09-25 20:15:26,302][96886] Decorrelating experience for 0 frames...
+[2023-09-25 20:15:26,306][96884] Decorrelating experience for 0 frames...
+[2023-09-25 20:15:26,306][96889] Decorrelating experience for 0 frames...
+[2023-09-25 20:15:26,307][96882] Decorrelating experience for 0 frames...
+[2023-09-25 20:15:26,307][96888] Decorrelating experience for 0 frames...
+[2023-09-25 20:15:26,309][96868] Decorrelating experience for 0 frames...
+[2023-09-25 20:15:26,310][96885] Decorrelating experience for 0 frames...
+[2023-09-25 20:15:26,312][96887] Decorrelating experience for 0 frames...
+[2023-09-25 20:15:30,710][95689] Fps is (10 sec: 1638.4, 60 sec: 1638.4, 300 sec: 1638.4). Total num frames: 8192. Throughput: 0: 204.8, 1: 204.8. Samples: 2048. Policy #0 lag: (min: 4.0, avg: 4.0, max: 4.0)
+[2023-09-25 20:15:35,710][95689] Fps is (10 sec: 3276.8, 60 sec: 3276.8, 300 sec: 3276.8). Total num frames: 32768. Throughput: 0: 398.0, 1: 406.3. Samples: 8043. Policy #0 lag: (min: 15.0, avg: 15.0, max: 15.0)
+[2023-09-25 20:15:35,711][95689] Avg episode reward: [(0, '2.667'), (1, '7.000')]
+[2023-09-25 20:15:39,794][95689] Heartbeat connected on Batcher_0
+[2023-09-25 20:15:39,797][95689] Heartbeat connected on LearnerWorker_p0
+[2023-09-25 20:15:39,800][95689] Heartbeat connected on Batcher_1
+[2023-09-25 20:15:39,802][95689] Heartbeat connected on LearnerWorker_p1
+[2023-09-25 20:15:39,809][95689] Heartbeat connected on InferenceWorker_p0-w0
+[2023-09-25 20:15:39,812][95689] Heartbeat connected on InferenceWorker_p1-w0
+[2023-09-25 20:15:39,813][95689] Heartbeat connected on RolloutWorker_w0
+[2023-09-25 20:15:39,818][95689] Heartbeat connected on RolloutWorker_w1
+[2023-09-25 20:15:39,819][95689] Heartbeat connected on RolloutWorker_w2
+[2023-09-25 20:15:39,822][95689] Heartbeat connected on RolloutWorker_w3
+[2023-09-25 20:15:39,826][95689] Heartbeat connected on RolloutWorker_w4
+[2023-09-25 20:15:39,827][95689] Heartbeat connected on RolloutWorker_w5
+[2023-09-25 20:15:39,830][95689] Heartbeat connected on RolloutWorker_w6
+[2023-09-25 20:15:39,835][95689] Heartbeat connected on RolloutWorker_w7
+[2023-09-25 20:15:40,710][95689] Fps is (10 sec: 5734.3, 60 sec: 4369.1, 300 sec: 4369.1). Total num frames: 65536. Throughput: 0: 414.1, 1: 419.9. Samples: 12511. Policy #0 lag: (min: 14.0, avg: 14.0, max: 14.0)
+[2023-09-25 20:15:40,711][95689] Avg episode reward: [(0, '2.611'), (1, '4.600')]
+[2023-09-25 20:15:42,747][96848] Updated weights for policy 0, policy_version 160 (0.0018)
+[2023-09-25 20:15:42,748][96849] Updated weights for policy 1, policy_version 160 (0.0017)
+[2023-09-25 20:15:45,710][95689] Fps is (10 sec: 6553.6, 60 sec: 4915.2, 300 sec: 4915.2). Total num frames: 98304. Throughput: 0: 563.0, 1: 563.2. Samples: 22524. Policy #0 lag: (min: 15.0, avg: 15.0, max: 15.0)
+[2023-09-25 20:15:45,711][95689] Avg episode reward: [(0, '2.586'), (1, '4.250')]
+[2023-09-25 20:15:50,710][95689] Fps is (10 sec: 6553.6, 60 sec: 5242.9, 300 sec: 5242.9). Total num frames: 131072. Throughput: 0: 638.3, 1: 641.1. Samples: 31985. Policy #0 lag: (min: 13.0, avg: 13.0, max: 13.0)
+[2023-09-25 20:15:50,711][95689] Avg episode reward: [(0, '2.686'), (1, '4.310')]
+[2023-09-25 20:15:55,667][96849] Updated weights for policy 1, policy_version 320 (0.0017)
+[2023-09-25 20:15:55,667][96848] Updated weights for policy 0, policy_version 320 (0.0017)
+[2023-09-25 20:15:55,710][95689] Fps is (10 sec: 6553.6, 60 sec: 5461.3, 300 sec: 5461.3). Total num frames: 163840. Throughput: 0: 613.2, 1: 614.4. Samples: 36827. Policy #0 lag: (min: 4.0, avg: 4.0, max: 4.0)
+[2023-09-25 20:15:55,711][95689] Avg episode reward: [(0, '2.628'), (1, '3.974')]
+[2023-09-25 20:16:00,710][95689] Fps is (10 sec: 5734.4, 60 sec: 5383.3, 300 sec: 5383.3). Total num frames: 188416. Throughput: 0: 656.8, 1: 659.3. Samples: 46063. Policy #0 lag: (min: 15.0, avg: 15.0, max: 15.0)
+[2023-09-25 20:16:00,711][95689] Avg episode reward: [(0, '2.648'), (1, '3.857')]
+[2023-09-25 20:16:05,710][95689] Fps is (10 sec: 5734.4, 60 sec: 5529.6, 300 sec: 5529.6). Total num frames: 221184. Throughput: 0: 693.5, 1: 695.9. Samples: 55579. Policy #0 lag: (min: 7.0, avg: 7.0, max: 7.0)
+[2023-09-25 20:16:05,711][95689] Avg episode reward: [(0, '2.633'), (1, '3.614')]
+[2023-09-25 20:16:05,869][96647] Saving new best policy, reward=2.633!
+[2023-09-25 20:16:05,899][96710] Saving new best policy, reward=3.614!
+[2023-09-25 20:16:08,456][96849] Updated weights for policy 1, policy_version 480 (0.0016)
+[2023-09-25 20:16:08,457][96848] Updated weights for policy 0, policy_version 480 (0.0018)
+[2023-09-25 20:16:10,710][95689] Fps is (10 sec: 6553.7, 60 sec: 5643.4, 300 sec: 5643.4). Total num frames: 253952. Throughput: 0: 671.6, 1: 674.6. Samples: 60581. Policy #0 lag: (min: 12.0, avg: 12.0, max: 12.0)
+[2023-09-25 20:16:10,711][95689] Avg episode reward: [(0, '2.697'), (1, '3.672')]
+[2023-09-25 20:16:10,711][96647] Saving new best policy, reward=2.697!
+[2023-09-25 20:16:10,711][96710] Saving new best policy, reward=3.672!
+[2023-09-25 20:16:15,710][95689] Fps is (10 sec: 6553.6, 60 sec: 5734.4, 300 sec: 5734.4). Total num frames: 286720. Throughput: 0: 751.0, 1: 752.4. Samples: 69703. Policy #0 lag: (min: 8.0, avg: 8.0, max: 8.0)
+[2023-09-25 20:16:15,711][95689] Avg episode reward: [(0, '2.747'), (1, '3.533')]
+[2023-09-25 20:16:15,712][96647] Saving new best policy, reward=2.747!
+[2023-09-25 20:16:20,710][95689] Fps is (10 sec: 6553.4, 60 sec: 5808.9, 300 sec: 5808.9). Total num frames: 319488. Throughput: 0: 790.7, 1: 790.0. Samples: 79176. Policy #0 lag: (min: 15.0, avg: 15.0, max: 15.0)
+[2023-09-25 20:16:20,711][95689] Avg episode reward: [(0, '2.695'), (1, '3.565')]
+[2023-09-25 20:16:21,656][96849] Updated weights for policy 1, policy_version 640 (0.0019)
+[2023-09-25 20:16:21,656][96848] Updated weights for policy 0, policy_version 640 (0.0017)
+[2023-09-25 20:16:25,710][95689] Fps is (10 sec: 6553.6, 60 sec: 5870.9, 300 sec: 5870.9). Total num frames: 352256. Throughput: 0: 794.9, 1: 793.0. Samples: 83968. Policy #0 lag: (min: 15.0, avg: 15.0, max: 15.0)
+[2023-09-25 20:16:25,711][95689] Avg episode reward: [(0, '2.814'), (1, '3.484')]
+[2023-09-25 20:16:25,712][96647] Saving new best policy, reward=2.814!
+[2023-09-25 20:16:30,710][95689] Fps is (10 sec: 6144.1, 60 sec: 6212.3, 300 sec: 5860.4). Total num frames: 380928. Throughput: 0: 786.3, 1: 789.4. Samples: 93429. Policy #0 lag: (min: 15.0, avg: 15.0, max: 15.0)
+[2023-09-25 20:16:30,711][95689] Avg episode reward: [(0, '2.966'), (1, '3.390')]
+[2023-09-25 20:16:30,731][96647] Saving new best policy, reward=2.966!
+[2023-09-25 20:16:34,554][96849] Updated weights for policy 1, policy_version 800 (0.0014)
+[2023-09-25 20:16:34,555][96848] Updated weights for policy 0, policy_version 800 (0.0018)
+[2023-09-25 20:16:35,710][95689] Fps is (10 sec: 5734.4, 60 sec: 6280.5, 300 sec: 5851.4). Total num frames: 409600. Throughput: 0: 787.2, 1: 788.1. Samples: 102873. Policy #0 lag: (min: 15.0, avg: 15.0, max: 15.0)
+[2023-09-25 20:16:35,711][95689] Avg episode reward: [(0, '2.927'), (1, '3.360')]
+[2023-09-25 20:16:40,710][95689] Fps is (10 sec: 6144.0, 60 sec: 6280.5, 300 sec: 5898.2). Total num frames: 442368. Throughput: 0: 790.2, 1: 790.4. Samples: 107958. Policy #0 lag: (min: 6.0, avg: 6.0, max: 6.0)
+[2023-09-25 20:16:40,711][95689] Avg episode reward: [(0, '2.860'), (1, '3.280')]
+[2023-09-25 20:16:41,784][95689] Keyboard interrupt detected in the event loop EvtLoop [Runner_EvtLoop, process=main process 95689], exiting...
+[2023-09-25 20:16:41,784][96885] Stopping RolloutWorker_w3...
+[2023-09-25 20:16:41,784][96884] Stopping RolloutWorker_w2...
+[2023-09-25 20:16:41,784][96886] Stopping RolloutWorker_w4...
+[2023-09-25 20:16:41,784][96882] Stopping RolloutWorker_w1...
+[2023-09-25 20:16:41,785][96868] Stopping RolloutWorker_w0...
+[2023-09-25 20:16:41,785][96885] Loop rollout_proc3_evt_loop terminating...
+[2023-09-25 20:16:41,784][95689] Runner profile tree view:
+main_loop: 81.9507
+[2023-09-25 20:16:41,785][96884] Loop rollout_proc2_evt_loop terminating...
+[2023-09-25 20:16:41,785][96868] Loop rollout_proc0_evt_loop terminating...
+[2023-09-25 20:16:41,785][96886] Loop rollout_proc4_evt_loop terminating...
+[2023-09-25 20:16:41,785][96882] Loop rollout_proc1_evt_loop terminating...
+[2023-09-25 20:16:41,785][95689] Collected {0: 225280, 1: 225280}, FPS: 5497.9
+[2023-09-25 20:16:41,785][96888] Stopping RolloutWorker_w6...
+[2023-09-25 20:16:41,785][96889] Stopping RolloutWorker_w7...
+[2023-09-25 20:16:41,785][96888] Loop rollout_proc6_evt_loop terminating...
+[2023-09-25 20:16:41,785][96710] Stopping Batcher_1...
+[2023-09-25 20:16:41,785][96887] Stopping RolloutWorker_w5...
+[2023-09-25 20:16:41,785][96889] Loop rollout_proc7_evt_loop terminating...
+[2023-09-25 20:16:41,786][96887] Loop rollout_proc5_evt_loop terminating...
+[2023-09-25 20:16:41,785][96647] Stopping Batcher_0...
+[2023-09-25 20:16:41,786][96710] Loop batcher_evt_loop terminating...
+[2023-09-25 20:16:41,786][96647] Loop batcher_evt_loop terminating...
+[2023-09-25 20:16:41,787][96710] Saving ./train_atari/atari_beamrider/checkpoint_p1/checkpoint_000000880_225280.pth...
+[2023-09-25 20:16:41,787][96647] Saving ./train_atari/atari_beamrider/checkpoint_p0/checkpoint_000000880_225280.pth...
+[2023-09-25 20:16:41,800][96849] Weights refcount: 2 0
+[2023-09-25 20:16:41,800][96848] Weights refcount: 2 0
+[2023-09-25 20:16:41,801][96849] Stopping InferenceWorker_p1-w0...
+[2023-09-25 20:16:41,801][96848] Stopping InferenceWorker_p0-w0...
+[2023-09-25 20:16:41,801][96849] Loop inference_proc1-0_evt_loop terminating...
+[2023-09-25 20:16:41,801][96848] Loop inference_proc0-0_evt_loop terminating...
+[2023-09-25 20:16:41,824][96710] Stopping LearnerWorker_p1...
+[2023-09-25 20:16:41,824][96710] Loop learner_proc1_evt_loop terminating...
+[2023-09-25 20:16:41,825][96647] Stopping LearnerWorker_p0...
+[2023-09-25 20:16:41,825][96647] Loop learner_proc0_evt_loop terminating...