First Push

SoccerTwos.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d28128f7eee0d6b1dc4f86d3215d0c56af0cf20642df7a5e0d4bd82dd6b7642
+oid sha256:6d0b23134e531515a56956b4af49c9d0e916b8f497a0212abf0d891572b0063e
 size 1768747

SoccerTwos/SoccerTwos-2999995.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d0b23134e531515a56956b4af49c9d0e916b8f497a0212abf0d891572b0063e
+size 1768747

SoccerTwos/SoccerTwos-2999995.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e21ae41439dbccd87976ca09fff9a8314b090fd7368ff3cc02c1b1fae4274f42
+size 28431567

SoccerTwos/SoccerTwos-3010466.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d0b23134e531515a56956b4af49c9d0e916b8f497a0212abf0d891572b0063e
+size 1768747

SoccerTwos/SoccerTwos-3010466.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c9b6b012e735f6fc47221913d26feaade4014e020d7f97588e5f140b8cfed98
+size 28431567

SoccerTwos/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:870e0aff3f2c94e970cf4440b1033e475e6fcd10a4ea5b3335cb3e0cd340adc9
+oid sha256:c19acba98fd7a2d4f4ea5058ddc4305dc42852860250341b295f14d1a86e74b0
 size 28430591

SoccerTwos/events.out.tfevents.1692326814.negocia-DGX-Station-A100-920-23487-2530-000.3238631.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c6f74e42f6c03c1dda1a8bad222917a36aa34f0c4e60fe3fde292d7ca3af75a
-size 2421123
+oid sha256:82a2b1ad3a75d2edfaa31257a916852144daf255b4b1b4299e15ae3332cd2e9a
+size 2564739

run_logs/Player-0.log

@@ -723,3 +723,770 @@ TOO LONG
 TOO LONG
 TOO LONG
 TOO LONG
+Setting up 64 worker threads for Enlighten.
+Memory Statistics:
+[ALLOC_TEMP_TLS] TLS Allocator
+  StackAllocators : 
+    [ALLOC_TEMP_MAIN]
+      Peak usage frame count: [8.0 KB-16.0 KB]: 57706 frames, [2.0 MB-4.0 MB]: 1 frames
+      Initial Block Size 4.0 MB
+      Current Block Size 4.0 MB
+      Peak Allocated Bytes 2.1 MB
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 18]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 42]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 58]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 44]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 73]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 8]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 40]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 36]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 77]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 16]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 10]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 4]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 27]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 6]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 52]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 60]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 110]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 85]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 103]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 70]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 79]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 1]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 78]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 66]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 74]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 101]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 53]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 38]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 11]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 3]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 20]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 46]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_BatchDeleteObjects]
+      Initial Block Size 64.0 KB
+      Current Block Size 64.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 43]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 96]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0.7 KB
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 106]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 30]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 51]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 63]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 26]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 21]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 82]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 86]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 25]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 39]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 100]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 111]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 92]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 31]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 71]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 47]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 29]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 13]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 15]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 55]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 76]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 98]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 7]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 22]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 80]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 75]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 35]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 67]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 2]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 8]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 90]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 59]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 23]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 1]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 32]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 33]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 49]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 10]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 24]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 15]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 12]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 83]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Loading.PreloadManager]
+      Initial Block Size 256.0 KB
+      Current Block Size 300.0 KB
+      Peak Allocated Bytes 249.6 KB
+      Overflow Count 4
+    [ALLOC_TEMP_Job.Worker 107]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 87]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 99]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 109]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 108]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 14]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 65]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 97]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 3]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 17]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 12]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_AssetGarbageCollectorHelper] x 127
+      Initial Block Size 64.0 KB
+      Current Block Size 64.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 5]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 56]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 13]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 88]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 62]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 84]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 37]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 50]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 54]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 2]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 94]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 0]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 41]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 11]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 4]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 81]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 6]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 0]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 9]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 5]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 64]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 14]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 91]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 57]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 95]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 48]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 68]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 34]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 89]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 72]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_EnlightenWorker] x 64
+      Initial Block Size 64.0 KB
+      Current Block Size 64.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 104]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 28]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 69]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 61]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 9]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 102]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 93]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 7]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 105]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 19]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 45]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
+    [ALLOC_TEMP_Loading.AsyncRead]
+      Initial Block Size 64.0 KB
+      Current Block Size 64.0 KB
+      Peak Allocated Bytes 128 B
+      Overflow Count 0
+[ALLOC_DEFAULT] Dual Thread Allocator
+  Peak main deferred allocation count 4
+    [ALLOC_BUCKET]
+      Large Block size 4.0 MB
+      Used Block count 1
+      Peak Allocated bytes 1.0 MB
+    [ALLOC_DEFAULT_MAIN]
+      Peak usage frame count: [4.0 MB-8.0 MB]: 57707 frames
+      Requested Block Size 16.0 MB
+      Peak Block count 1
+      Peak Allocated memory 4.8 MB
+      Peak Large allocation bytes 0 B
+    [ALLOC_DEFAULT_THREAD]
+      Peak usage frame count: [16.0 MB-32.0 MB]: 57707 frames
+      Requested Block Size 16.0 MB
+      Peak Block count 1
+      Peak Allocated memory 17.5 MB
+      Peak Large allocation bytes 16.0 MB
+[ALLOC_TEMP_JOB_1_FRAME]
+  Initial Block Size 2.0 MB
+  Used Block Count 1
+  Overflow Count (too large) 0
+  Overflow Count (full) 0
+[ALLOC_TEMP_JOB_2_FRAMES]
+  Initial Block Size 2.0 MB
+  Used Block Count 1
+  Overflow Count (too large) 0
+  Overflow Count (full) 0
+[ALLOC_TEMP_JOB_4_FRAMES (JobTemp)]
+  Initial Block Size 2.0 MB
+  Used Block Count 2
+  Overflow Count (too large) 0
+  Overflow Count (full) 0
+[ALLOC_TEMP_JOB_ASYNC (Background)]
+  Initial Block Size 1.0 MB
+  Used Block Count 3
+  Overflow Count (too large) 0
+  Overflow Count (full) 0
+[ALLOC_GFX] Dual Thread Allocator
+  Peak main deferred allocation count 0
+    [ALLOC_BUCKET]
+      Large Block size 4.0 MB
+      Used Block count 1
+      Peak Allocated bytes 1.0 MB
+    [ALLOC_GFX_MAIN]
+      Peak usage frame count: [32.0 KB-64.0 KB]: 57706 frames, [64.0 KB-128.0 KB]: 1 frames
+      Requested Block Size 16.0 MB
+      Peak Block count 1
+      Peak Allocated memory 65.6 KB
+      Peak Large allocation bytes 0 B
+    [ALLOC_GFX_THREAD]
+      Peak usage frame count: [128.0 KB-256.0 KB]: 57707 frames
+      Requested Block Size 16.0 MB
+      Peak Block count 1
+      Peak Allocated memory 173.5 KB
+      Peak Large allocation bytes 0 B
+[ALLOC_CACHEOBJECTS] Dual Thread Allocator
+  Peak main deferred allocation count 0
+    [ALLOC_BUCKET]
+      Large Block size 4.0 MB
+      Used Block count 1
+      Peak Allocated bytes 1.0 MB
+    [ALLOC_CACHEOBJECTS_MAIN]
+      Peak usage frame count: [0.5 MB-1.0 MB]: 57707 frames
+      Requested Block Size 4.0 MB
+      Peak Block count 1
+      Peak Allocated memory 0.6 MB
+      Peak Large allocation bytes 0 B
+    [ALLOC_CACHEOBJECTS_THREAD]
+      Peak usage frame count: [0.5 MB-1.0 MB]: 57706 frames, [4.0 MB-8.0 MB]: 1 frames
+      Requested Block Size 4.0 MB
+      Peak Block count 2
+      Peak Allocated memory 4.5 MB
+      Peak Large allocation bytes 0 B
+[ALLOC_TYPETREE] Dual Thread Allocator
+  Peak main deferred allocation count 0
+    [ALLOC_BUCKET]
+      Large Block size 4.0 MB
+      Used Block count 1
+      Peak Allocated bytes 1.0 MB
+    [ALLOC_TYPETREE_MAIN]
+      Peak usage frame count: [0-1.0 KB]: 57707 frames
+      Requested Block Size 2.0 MB
+      Peak Block count 1
+      Peak Allocated memory 1.0 KB
+      Peak Large allocation bytes 0 B
+    [ALLOC_TYPETREE_THREAD]
+      Peak usage frame count: [1.0 KB-2.0 KB]: 57706 frames, [2.0 KB-4.0 KB]: 1 frames
+      Requested Block Size 2.0 MB
+      Peak Block count 1
+      Peak Allocated memory 2.2 KB
+      Peak Large allocation bytes 0 B

run_logs/timers.json

@@ -2,261 +2,261 @@
     "name": "root",
     "gauges": {
         "SoccerTwos.Policy.Entropy.mean": {
-            "value": 3.2222719192504883,
-            "min": 3.18680739402771,
-            "max": 3.295748710632324,
-            "count": 50
+            "value": 2.2744622230529785,
+            "min": 2.232063055038452,
+            "max": 3.2957096099853516,
+            "count": 301
         },
         "SoccerTwos.Policy.Entropy.sum": {
-            "value": 57124.4375,
-            "min": 15960.603515625,
-            "max": 105463.9609375,
-            "count": 50
+            "value": 45416.4609375,
+            "min": 24416.8046875,
+            "max": 143192.609375,
+            "count": 301
         },
         "SoccerTwos.Environment.EpisodeLength.mean": {
-            "value": 882.1666666666666,
-            "min": 406.54545454545456,
+            "value": 48.59,
+            "min": 45.02803738317757,
             "max": 999.0,
-            "count": 50
+            "count": 301
         },
         "SoccerTwos.Environment.EpisodeLength.sum": {
-            "value": 21172.0,
-            "min": 13836.0,
-            "max": 27164.0,
-            "count": 50
+            "value": 19436.0,
+            "min": 10896.0,
+            "max": 30116.0,
+            "count": 301
         },
         "SoccerTwos.Self-play.ELO.mean": {
-            "value": 1193.6615718716212,
-            "min": 1193.6615718716212,
-            "max": 1202.4290909239016,
-            "count": 44
+            "value": 1439.2786878879729,
+            "min": 1188.561030279781,
+            "max": 1450.550956658333,
+            "count": 275
         },
         "SoccerTwos.Self-play.ELO.sum": {
-            "value": 4774.646287486485,
-            "min": 2389.9051923307993,
-            "max": 14420.734528666535,
-            "count": 44
+            "value": 287855.7375775946,
+            "min": 2378.992766479224,
+            "max": 314924.30464004644,
+            "count": 275
         },
         "SoccerTwos.Step.mean": {
-            "value": 499946.0,
-            "min": 9602.0,
-            "max": 499946.0,
-            "count": 50
+            "value": 3009970.0,
+            "min": 9076.0,
+            "max": 3009970.0,
+            "count": 301
         },
         "SoccerTwos.Step.sum": {
-            "value": 499946.0,
-            "min": 9602.0,
-            "max": 499946.0,
-            "count": 50
+            "value": 3009970.0,
+            "min": 9076.0,
+            "max": 3009970.0,
+            "count": 301
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.mean": {
-            "value": -0.01228646282106638,
-            "min": -0.032066188752651215,
-            "max": 0.005078669171780348,
-            "count": 50
+            "value": 0.00556741701439023,
+            "min": -0.037207089364528656,
+            "max": 0.16740134358406067,
+            "count": 301
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.sum": {
-            "value": -0.14743755757808685,
-            "min": -0.5129154920578003,
-            "max": 0.05586536228656769,
-            "count": 50
+            "value": 1.1134834289550781,
+            "min": -5.9903411865234375,
+            "max": 28.458227157592773,
+            "count": 301
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.mean": {
-            "value": -0.014418837614357471,
-            "min": -0.03204234689474106,
-            "max": 0.0033151486422866583,
-            "count": 50
+            "value": 0.007489514537155628,
+            "min": -0.03448658809065819,
+            "max": 0.16675932705402374,
+            "count": 301
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.sum": {
-            "value": -0.17302605509757996,
-            "min": -0.5124354362487793,
-            "max": 0.043096933513879776,
-            "count": 50
+            "value": 1.4979028701782227,
+            "min": -5.9887590408325195,
+            "max": 28.34908676147461,
+            "count": 301
         },
         "SoccerTwos.Environment.CumulativeReward.mean": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 50
+            "count": 301
         },
         "SoccerTwos.Environment.CumulativeReward.sum": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 50
+            "count": 301
         },
         "SoccerTwos.Policy.ExtrinsicReward.mean": {
-            "value": -0.12416666746139526,
-            "min": -0.5310933334132035,
-            "max": 0.28889230581430286,
-            "count": 50
+            "value": -0.026724000573158265,
+            "min": -0.6666666666666666,
+            "max": 0.4395599941412608,
+            "count": 301
         },
         "SoccerTwos.Policy.ExtrinsicReward.sum": {
-            "value": -1.4900000095367432,
-            "min": -7.966400001198053,
-            "max": 3.7555999755859375,
-            "count": 50
+            "value": -5.344800114631653,
+            "min": -36.470200181007385,
+            "max": 66.4987998008728,
+            "count": 301
         },
         "SoccerTwos.Environment.GroupCumulativeReward.mean": {
-            "value": -0.12416666746139526,
-            "min": -0.5310933334132035,
-            "max": 0.28889230581430286,
-            "count": 50
+            "value": -0.026724000573158265,
+            "min": -0.6666666666666666,
+            "max": 0.4395599941412608,
+            "count": 301
         },
         "SoccerTwos.Environment.GroupCumulativeReward.sum": {
-            "value": -1.4900000095367432,
-            "min": -7.966400001198053,
-            "max": 3.7555999755859375,
-            "count": 50
+            "value": -5.344800114631653,
+            "min": -36.470200181007385,
+            "max": 66.4987998008728,
+            "count": 301
         },
         "SoccerTwos.IsTraining.mean": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 50
+            "count": 301
         },
         "SoccerTwos.IsTraining.sum": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 50
+            "count": 301
         },
         "SoccerTwos.Losses.PolicyLoss.mean": {
-            "value": 0.019728626206051557,
-            "min": 0.011391046621429268,
-            "max": 0.023163473070599138,
-            "count": 23
+            "value": 0.01690223190623025,
+            "min": 0.011579616327071562,
+            "max": 0.023286711684583374,
+            "count": 142
         },
         "SoccerTwos.Losses.PolicyLoss.sum": {
-            "value": 0.019728626206051557,
-            "min": 0.011391046621429268,
-            "max": 0.023163473070599138,
-            "count": 23
+            "value": 0.01690223190623025,
+            "min": 0.011579616327071562,
+            "max": 0.023286711684583374,
+            "count": 142
         },
         "SoccerTwos.Losses.ValueLoss.mean": {
-            "value": 0.0034201507456600665,
-            "min": 0.0008259918860858307,
-            "max": 0.0051478686897705,
-            "count": 23
+            "value": 0.10111841162045797,
+            "min": 2.272657062955356e-05,
+            "max": 0.10499386588732401,
+            "count": 142
         },
         "SoccerTwos.Losses.ValueLoss.sum": {
-            "value": 0.0034201507456600665,
-            "min": 0.0008259918860858307,
-            "max": 0.0051478686897705,
-            "count": 23
+            "value": 0.10111841162045797,
+            "min": 2.272657062955356e-05,
+            "max": 0.10499386588732401,
+            "count": 142
         },
         "SoccerTwos.Losses.BaselineLoss.mean": {
-            "value": 0.003457886151348551,
-            "min": 0.0008355041247947762,
-            "max": 0.005449820651362339,
-            "count": 23
+            "value": 0.10573899472753207,
+            "min": 2.0924207122637504e-05,
+            "max": 0.10958060696721077,
+            "count": 142
         },
         "SoccerTwos.Losses.BaselineLoss.sum": {
-            "value": 0.003457886151348551,
-            "min": 0.0008355041247947762,
-            "max": 0.005449820651362339,
-            "count": 23
+            "value": 0.10573899472753207,
+            "min": 2.0924207122637504e-05,
+            "max": 0.10958060696721077,
+            "count": 142
         },
         "SoccerTwos.Policy.LearningRate.mean": {
             "value": 0.0003,
             "min": 0.0003,
             "max": 0.0003,
-            "count": 23
+            "count": 142
         },
         "SoccerTwos.Policy.LearningRate.sum": {
             "value": 0.0003,
             "min": 0.0003,
             "max": 0.0003,
-            "count": 23
+            "count": 142
         },
         "SoccerTwos.Policy.Epsilon.mean": {
             "value": 0.20000000000000007,
-            "min": 0.2,
+            "min": 0.20000000000000007,
             "max": 0.20000000000000007,
-            "count": 23
+            "count": 142
         },
         "SoccerTwos.Policy.Epsilon.sum": {
             "value": 0.20000000000000007,
-            "min": 0.2,
+            "min": 0.20000000000000007,
             "max": 0.20000000000000007,
-            "count": 23
+            "count": 142
         },
         "SoccerTwos.Policy.Beta.mean": {
             "value": 0.005000000000000001,
-            "min": 0.005,
+            "min": 0.005000000000000001,
             "max": 0.005000000000000001,
-            "count": 23
+            "count": 142
         },
         "SoccerTwos.Policy.Beta.sum": {
             "value": 0.005000000000000001,
-            "min": 0.005,
+            "min": 0.005000000000000001,
             "max": 0.005000000000000001,
-            "count": 23
+            "count": 142
         }
     },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1692326065",
+        "start_time_seconds": "1692326814",
         "python_version": "3.9.17 | packaged by conda-forge | (main, Aug 10 2023, 07:02:31) \n[GCC 12.3.0]",
         "command_line_arguments": "/home/kotaishizuka/miniconda/envs/rl/bin/mlagents-learn ./config/poca/SoccerTwos.yaml --env=./training-envs-executables/SoccerTwos/SoccerTwos.x86_64 --run-id=SoccerTwos --no-graphics --force",
         "mlagents_version": "0.31.0.dev0",
         "mlagents_envs_version": "0.31.0.dev0",
         "communication_protocol_version": "1.5.0",
-        "pytorch_version": "2.0.1+cu117",
+        "pytorch_version": "1.13.0+cu117",
         "numpy_version": "1.21.2",
-        "end_time_seconds": "1692326743"
+        "end_time_seconds": "1692330797"
     },
-    "total": 678.8801201358438,
+    "total": 3983.5853504855186,
     "count": 1,
-    "self": 0.3215545192360878,
+    "self": 0.13617010228335857,
     "children": {
         "run_training.setup": {
-            "total": 0.010781640186905861,
+            "total": 0.010169833898544312,
             "count": 1,
-            "self": 0.010781640186905861
+            "self": 0.010169833898544312
         },
         "TrainerController.start_learning": {
-            "total": 678.5477839764208,
+            "total": 3983.4390105493367,
             "count": 1,
-            "self": 0.5137664936482906,
+            "self": 2.9237716384232044,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 1.7076039835810661,
-                    "count": 3,
-                    "self": 1.7076039835810661
+                    "total": 2.533159812912345,
+                    "count": 16,
+                    "self": 2.533159812912345
                 },
                 "TrainerController.advance": {
-                    "total": 676.1762186419219,
-                    "count": 32654,
-                    "self": 0.45217580534517765,
+                    "total": 3977.848170027137,
+                    "count": 200556,
+                    "self": 2.814035166054964,
                     "children": {
                         "env_step": {
-                            "total": 532.7235291823745,
-                            "count": 32654,
-                            "self": 402.25471655651927,
+                            "total": 3036.8756248299032,
+                            "count": 200556,
+                            "self": 2304.1390847805887,
                             "children": {
                                 "SubprocessEnvManager._take_step": {
-                                    "total": 130.16790090501308,
-                                    "count": 32654,
-                                    "self": 3.096593974158168,
+                                    "total": 730.966472717002,
+                                    "count": 200556,
+                                    "self": 18.915119621902704,
                                     "children": {
                                         "TorchPolicy.evaluate": {
-                                            "total": 127.07130693085492,
-                                            "count": 64874,
-                                            "self": 127.07130693085492
+                                            "total": 712.0513530950993,
+                                            "count": 384718,
+                                            "self": 712.0513530950993
                                         }
                                     }
                                 },
                                 "workers": {
-                                    "total": 0.30091172084212303,
-                                    "count": 32654,
+                                    "total": 1.7700673323124647,
+                                    "count": 200556,
                                     "self": 0.0,
                                     "children": {
                                         "worker_root": {
-                                            "total": 677.5487775281072,
-                                            "count": 32654,
+                                            "total": 3978.563171196729,
+                                            "count": 200556,
                                             "is_parallel": true,
-                                            "self": 343.6171973440796,
+                                            "self": 2073.00574160181,
                                             "children": {
                                                 "run_training.setup": {
                                                     "total": 0.0,
@@ -265,48 +265,48 @@
                                                     "self": 0.0,
                                                     "children": {
                                                         "steps_from_proto": {
-                                                            "total": 0.00227261520922184,
+                                                            "total": 0.002252792939543724,
                                                             "count": 2,
                                                             "is_parallel": true,
-                                                            "self": 0.00046112947165966034,
+                                                            "self": 0.0004422049969434738,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 0.0018114857375621796,
+                                                                    "total": 0.0018105879426002502,
                                                                     "count": 8,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0018114857375621796
+                                                                    "self": 0.0018105879426002502
                                                                 }
                                                             }
                                                         },
                                                         "UnityEnvironment.step": {
-                                                            "total": 0.023042185232043266,
+                                                            "total": 0.023576932027935982,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.0006137657910585403,
+                                                            "self": 0.0009839106351137161,
                                                             "children": {
                                                                 "UnityEnvironment._generate_step_input": {
-                                                                    "total": 0.0008106175810098648,
+                                                                    "total": 0.0008261147886514664,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0008106175810098648
+                                                                    "self": 0.0008261147886514664
                                                                 },
                                                                 "communicator.exchange": {
-                                                                    "total": 0.01983891800045967,
+                                                                    "total": 0.019841434434056282,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.01983891800045967
+                                                                    "self": 0.019841434434056282
                                                                 },
                                                                 "steps_from_proto": {
-                                                                    "total": 0.0017788838595151901,
+                                                                    "total": 0.0019254721701145172,
                                                                     "count": 2,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0003027338534593582,
+                                                                    "self": 0.0003166627138853073,
                                                                     "children": {
                                                                         "_process_rank_one_or_two_observation": {
-                                                                            "total": 0.001476150006055832,
+                                                                            "total": 0.00160880945622921,
                                                                             "count": 8,
                                                                             "is_parallel": true,
-                                                                            "self": 0.001476150006055832
+                                                                            "self": 0.00160880945622921
                                                                         }
                                                                     }
                                                                 }
@@ -315,50 +315,50 @@
                                                     }
                                                 },
                                                 "UnityEnvironment.step": {
-                                                    "total": 333.92770622111857,
-                                                    "count": 32653,
+                                                    "total": 1905.5294378641993,
+                                                    "count": 200555,
                                                     "is_parallel": true,
-                                                    "self": 19.874608892947435,
+                                                    "self": 115.9678994640708,
                                                     "children": {
                                                         "UnityEnvironment._generate_step_input": {
-                                                            "total": 13.80960377305746,
-                                                            "count": 32653,
+                                                            "total": 83.88343887589872,
+                                                            "count": 200555,
                                                             "is_parallel": true,
-                                                            "self": 13.80960377305746
+                                                            "self": 83.88343887589872
                                                         },
                                                         "communicator.exchange": {
-                                                            "total": 243.57853556983173,
-                                                            "count": 32653,
+                                                            "total": 1363.9825185704976,
+                                                            "count": 200555,
                                                             "is_parallel": true,
-                                                            "self": 243.57853556983173
+                                                            "self": 1363.9825185704976
                                                         },
                                                         "steps_from_proto": {
-                                                            "total": 56.664957985281944,
-                                                            "count": 65306,
+                                                            "total": 341.69558095373213,
+                                                            "count": 401110,
                                                             "is_parallel": true,
-                                                            "self": 8.49021466076374,
+                                                            "self": 51.02112327516079,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 48.174743324518204,
-                                                                    "count": 261224,
+                                                                    "total": 290.67445767857134,
+                                                                    "count": 1604440,
                                                                     "is_parallel": true,
-                                                                    "self": 48.174743324518204
+                                                                    "self": 290.67445767857134
                                                                 }
                                                             }
                                                         }
                                                     }
                                                 },
                                                 "steps_from_proto": {
-                                                    "total": 0.0038739629089832306,
-                                                    "count": 4,
+                                                    "total": 0.027991730719804764,
+                                                    "count": 30,
                                                     "is_parallel": true,
-                                                    "self": 0.000619182363152504,
+                                                    "self": 0.004335055127739906,
                                                     "children": {
                                                         "_process_rank_one_or_two_observation": {
-                                                            "total": 0.0032547805458307266,
-                                                            "count": 16,
+                                                            "total": 0.023656675592064857,
+                                                            "count": 120,
                                                             "is_parallel": true,
-                                                            "self": 0.0032547805458307266
+                                                            "self": 0.023656675592064857
                                                         }
                                                     }
                                                 }
@@ -369,31 +369,31 @@
                             }
                         },
                         "trainer_advance": {
-                            "total": 143.00051365420222,
-                            "count": 32654,
-                            "self": 4.759939888492227,
+                            "total": 938.1585100311786,
+                            "count": 200556,
+                            "self": 26.863946912810206,
                             "children": {
                                 "process_trajectory": {
-                                    "total": 36.244710413739085,
-                                    "count": 32654,
-                                    "self": 36.096403958275914,
+                                    "total": 296.42381161265075,
+                                    "count": 200556,
+                                    "self": 295.62428767606616,
                                     "children": {
                                         "RLTrainer._checkpoint": {
-                                            "total": 0.148306455463171,
-                                            "count": 1,
-                                            "self": 0.148306455463171
+                                            "total": 0.7995239365845919,
+                                            "count": 6,
+                                            "self": 0.7995239365845919
                                         }
                                     }
                                 },
                                 "_update_policy": {
-                                    "total": 101.99586335197091,
-                                    "count": 23,
-                                    "self": 74.7204381916672,
+                                    "total": 614.8707515057176,
+                                    "count": 143,
+                                    "self": 447.04196193255484,
                                     "children": {
                                         "TorchPOCAOptimizer.update": {
-                                            "total": 27.275425160303712,
-                                            "count": 693,
-                                            "self": 27.275425160303712
+                                            "total": 167.8287895731628,
+                                            "count": 4260,
+                                            "self": 167.8287895731628
                                         }
                                     }
                                 }
@@ -401,15 +401,20 @@
                         }
                     }
                 },
+                "trainer_threads": {
+                    "total": 8.21426510810852e-07,
+                    "count": 1,
+                    "self": 8.21426510810852e-07
+                },
                 "TrainerController._save_models": {
-                    "total": 0.15019485726952553,
+                    "total": 0.133908249437809,
                     "count": 1,
-                    "self": 2.9996037483215332e-05,
+                    "self": 0.001552453264594078,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.1501648612320423,
+                            "total": 0.1323557961732149,
                             "count": 1,
-                            "self": 0.1501648612320423
+                            "self": 0.1323557961732149
                         }
                     }
                 }

run_logs/training_status.json

@@ -1,10 +1,66 @@
 {
     "SoccerTwos": {
-        "elo": 1193.1690506485725
+        "checkpoints": [
+            {
+                "steps": 1499815,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-1499815.onnx",
+                "reward": 0.0,
+                "creation_time": 1692328776.5461588,
+                "auxillary_file_paths": [
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-1499815.pt"
+                ]
+            },
+            {
+                "steps": 1999968,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-1999968.onnx",
+                "reward": 0.0,
+                "creation_time": 1692329423.4035466,
+                "auxillary_file_paths": [
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-1999968.pt"
+                ]
+            },
+            {
+                "steps": 2499986,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-2499986.onnx",
+                "reward": 0.0,
+                "creation_time": 1692330098.0251286,
+                "auxillary_file_paths": [
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-2499986.pt"
+                ]
+            },
+            {
+                "steps": 2999995,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-2999995.onnx",
+                "reward": 0.0,
+                "creation_time": 1692330784.6261935,
+                "auxillary_file_paths": [
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-2999995.pt"
+                ]
+            },
+            {
+                "steps": 3010466,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-3010466.onnx",
+                "reward": null,
+                "creation_time": 1692330797.5487733,
+                "auxillary_file_paths": [
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-3010466.pt"
+                ]
+            }
+        ],
+        "elo": 1440.2787022605767,
+        "final_checkpoint": {
+            "steps": 3010466,
+            "file_path": "results/SoccerTwos/SoccerTwos.onnx",
+            "reward": null,
+            "creation_time": 1692330797.5487733,
+            "auxillary_file_paths": [
+                "results/SoccerTwos/SoccerTwos/SoccerTwos-3010466.pt"
+            ]
+        }
     },
     "metadata": {
         "stats_format_version": "0.3.0",
         "mlagents_version": "0.31.0.dev0",
-        "torch_version": "2.0.1+cu117"
+        "torch_version": "1.13.0+cu117"
     }
 }