Upload folder using huggingface_hub
Browse files
.summary/0/events.out.tfevents.1711742223.ip-172-31-79-185.ec2.internal
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1c34a56081bc81ce84e226e9f8b2602235817f949c8274fc23fd8b817ac47092
|
3 |
+
size 12345477
|
checkpoint_p0/checkpoint_000040822_668827648.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:bb6dbe9b2a66adcb5f2ab0741d05275356c3ecc791631f3e248a7ed6e1c0c587
|
3 |
+
size 76479020
|
checkpoint_p0/checkpoint_000041133_673923072.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:85a5c3311871836bdf554747b7296b8d858e6a846aee5f688918187c3f4f3d6d
|
3 |
+
size 76479020
|
sf_log.txt
CHANGED
@@ -13934,3 +13934,197 @@
|
|
13934 |
[2024-03-29 16:36:18,831][00497] Updated weights for policy 0, policy_version 40705 (0.0019)
|
13935 |
[2024-03-29 16:36:18,839][00126] Fps is (10 sec: 44236.8, 60 sec: 42325.4, 300 sec: 41932.0). Total num frames: 666910720. Throughput: 0: 41786.9. Samples: 549035600. Policy #0 lag: (min: 0.0, avg: 22.9, max: 41.0)
|
13936 |
[2024-03-29 16:36:18,840][00126] Avg episode reward: [(0, '0.473')]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
13934 |
[2024-03-29 16:36:18,831][00497] Updated weights for policy 0, policy_version 40705 (0.0019)
|
13935 |
[2024-03-29 16:36:18,839][00126] Fps is (10 sec: 44236.8, 60 sec: 42325.4, 300 sec: 41932.0). Total num frames: 666910720. Throughput: 0: 41786.9. Samples: 549035600. Policy #0 lag: (min: 0.0, avg: 22.9, max: 41.0)
|
13936 |
[2024-03-29 16:36:18,840][00126] Avg episode reward: [(0, '0.473')]
|
13937 |
+
[2024-03-29 16:36:23,144][00497] Updated weights for policy 0, policy_version 40715 (0.0022)
|
13938 |
+
[2024-03-29 16:36:23,839][00126] Fps is (10 sec: 40960.4, 60 sec: 42052.3, 300 sec: 41987.5). Total num frames: 667107328. Throughput: 0: 41841.3. Samples: 549306280. Policy #0 lag: (min: 0.0, avg: 22.9, max: 41.0)
|
13939 |
+
[2024-03-29 16:36:23,840][00126] Avg episode reward: [(0, '0.504')]
|
13940 |
+
[2024-03-29 16:36:26,762][00497] Updated weights for policy 0, policy_version 40725 (0.0017)
|
13941 |
+
[2024-03-29 16:36:28,839][00126] Fps is (10 sec: 42597.5, 60 sec: 41779.1, 300 sec: 42043.0). Total num frames: 667336704. Throughput: 0: 41923.6. Samples: 549554060. Policy #0 lag: (min: 0.0, avg: 22.9, max: 41.0)
|
13942 |
+
[2024-03-29 16:36:28,840][00126] Avg episode reward: [(0, '0.561')]
|
13943 |
+
[2024-03-29 16:36:30,165][00497] Updated weights for policy 0, policy_version 40735 (0.0019)
|
13944 |
+
[2024-03-29 16:36:33,839][00126] Fps is (10 sec: 42598.4, 60 sec: 42052.4, 300 sec: 41987.5). Total num frames: 667533312. Throughput: 0: 41960.9. Samples: 549675020. Policy #0 lag: (min: 0.0, avg: 22.9, max: 41.0)
|
13945 |
+
[2024-03-29 16:36:33,840][00126] Avg episode reward: [(0, '0.493')]
|
13946 |
+
[2024-03-29 16:36:34,373][00497] Updated weights for policy 0, policy_version 40745 (0.0025)
|
13947 |
+
[2024-03-29 16:36:38,608][00497] Updated weights for policy 0, policy_version 40755 (0.0022)
|
13948 |
+
[2024-03-29 16:36:38,839][00126] Fps is (10 sec: 39321.6, 60 sec: 42052.2, 300 sec: 42043.0). Total num frames: 667729920. Throughput: 0: 41917.4. Samples: 549939920. Policy #0 lag: (min: 0.0, avg: 22.9, max: 41.0)
|
13949 |
+
[2024-03-29 16:36:38,840][00126] Avg episode reward: [(0, '0.444')]
|
13950 |
+
[2024-03-29 16:36:42,322][00497] Updated weights for policy 0, policy_version 40765 (0.0025)
|
13951 |
+
[2024-03-29 16:36:43,839][00126] Fps is (10 sec: 44236.6, 60 sec: 42052.4, 300 sec: 42098.5). Total num frames: 667975680. Throughput: 0: 42078.2. Samples: 550191700. Policy #0 lag: (min: 1.0, avg: 20.0, max: 42.0)
|
13952 |
+
[2024-03-29 16:36:43,840][00126] Avg episode reward: [(0, '0.558')]
|
13953 |
+
[2024-03-29 16:36:45,694][00497] Updated weights for policy 0, policy_version 40775 (0.0027)
|
13954 |
+
[2024-03-29 16:36:45,717][00476] Signal inference workers to stop experience collection... (19600 times)
|
13955 |
+
[2024-03-29 16:36:45,753][00497] InferenceWorker_p0-w0: stopping experience collection (19600 times)
|
13956 |
+
[2024-03-29 16:36:45,942][00476] Signal inference workers to resume experience collection... (19600 times)
|
13957 |
+
[2024-03-29 16:36:45,943][00497] InferenceWorker_p0-w0: resuming experience collection (19600 times)
|
13958 |
+
[2024-03-29 16:36:48,839][00126] Fps is (10 sec: 44237.4, 60 sec: 42325.3, 300 sec: 42043.0). Total num frames: 668172288. Throughput: 0: 42035.6. Samples: 550307620. Policy #0 lag: (min: 1.0, avg: 20.0, max: 42.0)
|
13959 |
+
[2024-03-29 16:36:48,840][00126] Avg episode reward: [(0, '0.550')]
|
13960 |
+
[2024-03-29 16:36:50,026][00497] Updated weights for policy 0, policy_version 40785 (0.0032)
|
13961 |
+
[2024-03-29 16:36:53,839][00126] Fps is (10 sec: 39321.7, 60 sec: 42052.4, 300 sec: 42043.0). Total num frames: 668368896. Throughput: 0: 42268.8. Samples: 550580900. Policy #0 lag: (min: 1.0, avg: 20.0, max: 42.0)
|
13962 |
+
[2024-03-29 16:36:53,840][00126] Avg episode reward: [(0, '0.566')]
|
13963 |
+
[2024-03-29 16:36:54,024][00497] Updated weights for policy 0, policy_version 40795 (0.0020)
|
13964 |
+
[2024-03-29 16:36:57,791][00497] Updated weights for policy 0, policy_version 40805 (0.0020)
|
13965 |
+
[2024-03-29 16:36:58,839][00126] Fps is (10 sec: 42598.3, 60 sec: 42325.3, 300 sec: 42043.0). Total num frames: 668598272. Throughput: 0: 42390.3. Samples: 550827920. Policy #0 lag: (min: 1.0, avg: 20.0, max: 42.0)
|
13966 |
+
[2024-03-29 16:36:58,840][00126] Avg episode reward: [(0, '0.518')]
|
13967 |
+
[2024-03-29 16:37:01,221][00497] Updated weights for policy 0, policy_version 40815 (0.0029)
|
13968 |
+
[2024-03-29 16:37:03,839][00126] Fps is (10 sec: 44236.7, 60 sec: 42325.3, 300 sec: 42043.0). Total num frames: 668811264. Throughput: 0: 42280.3. Samples: 550938220. Policy #0 lag: (min: 1.0, avg: 20.0, max: 42.0)
|
13969 |
+
[2024-03-29 16:37:03,840][00126] Avg episode reward: [(0, '0.565')]
|
13970 |
+
[2024-03-29 16:37:04,087][00476] Saving /workspace/metta/train_dir/b.a20.20x20_40x40.norm/checkpoint_p0/checkpoint_000040822_668827648.pth...
|
13971 |
+
[2024-03-29 16:37:04,389][00476] Removing /workspace/metta/train_dir/b.a20.20x20_40x40.norm/checkpoint_p0/checkpoint_000040205_658718720.pth
|
13972 |
+
[2024-03-29 16:37:05,307][00497] Updated weights for policy 0, policy_version 40825 (0.0029)
|
13973 |
+
[2024-03-29 16:37:08,839][00126] Fps is (10 sec: 39321.6, 60 sec: 42052.2, 300 sec: 41987.5). Total num frames: 668991488. Throughput: 0: 42217.8. Samples: 551206080. Policy #0 lag: (min: 1.0, avg: 19.1, max: 42.0)
|
13974 |
+
[2024-03-29 16:37:08,841][00126] Avg episode reward: [(0, '0.510')]
|
13975 |
+
[2024-03-29 16:37:09,637][00497] Updated weights for policy 0, policy_version 40835 (0.0024)
|
13976 |
+
[2024-03-29 16:37:13,397][00497] Updated weights for policy 0, policy_version 40845 (0.0019)
|
13977 |
+
[2024-03-29 16:37:13,839][00126] Fps is (10 sec: 40960.1, 60 sec: 42052.3, 300 sec: 42043.0). Total num frames: 669220864. Throughput: 0: 42384.1. Samples: 551461340. Policy #0 lag: (min: 1.0, avg: 19.1, max: 42.0)
|
13978 |
+
[2024-03-29 16:37:13,840][00126] Avg episode reward: [(0, '0.519')]
|
13979 |
+
[2024-03-29 16:37:16,798][00497] Updated weights for policy 0, policy_version 40855 (0.0021)
|
13980 |
+
[2024-03-29 16:37:17,130][00476] Signal inference workers to stop experience collection... (19650 times)
|
13981 |
+
[2024-03-29 16:37:17,154][00497] InferenceWorker_p0-w0: stopping experience collection (19650 times)
|
13982 |
+
[2024-03-29 16:37:17,348][00476] Signal inference workers to resume experience collection... (19650 times)
|
13983 |
+
[2024-03-29 16:37:17,348][00497] InferenceWorker_p0-w0: resuming experience collection (19650 times)
|
13984 |
+
[2024-03-29 16:37:18,839][00126] Fps is (10 sec: 45875.2, 60 sec: 42325.3, 300 sec: 42098.5). Total num frames: 669450240. Throughput: 0: 42167.1. Samples: 551572540. Policy #0 lag: (min: 1.0, avg: 19.1, max: 42.0)
|
13985 |
+
[2024-03-29 16:37:18,840][00126] Avg episode reward: [(0, '0.554')]
|
13986 |
+
[2024-03-29 16:37:21,042][00497] Updated weights for policy 0, policy_version 40865 (0.0025)
|
13987 |
+
[2024-03-29 16:37:23,839][00126] Fps is (10 sec: 40959.3, 60 sec: 42052.2, 300 sec: 42043.0). Total num frames: 669630464. Throughput: 0: 42160.8. Samples: 551837160. Policy #0 lag: (min: 1.0, avg: 19.1, max: 42.0)
|
13988 |
+
[2024-03-29 16:37:23,840][00126] Avg episode reward: [(0, '0.464')]
|
13989 |
+
[2024-03-29 16:37:25,288][00497] Updated weights for policy 0, policy_version 40875 (0.0023)
|
13990 |
+
[2024-03-29 16:37:28,788][00497] Updated weights for policy 0, policy_version 40885 (0.0027)
|
13991 |
+
[2024-03-29 16:37:28,839][00126] Fps is (10 sec: 40959.7, 60 sec: 42052.3, 300 sec: 42098.6). Total num frames: 669859840. Throughput: 0: 42348.9. Samples: 552097400. Policy #0 lag: (min: 1.0, avg: 19.1, max: 42.0)
|
13992 |
+
[2024-03-29 16:37:28,841][00126] Avg episode reward: [(0, '0.526')]
|
13993 |
+
[2024-03-29 16:37:32,101][00497] Updated weights for policy 0, policy_version 40895 (0.0029)
|
13994 |
+
[2024-03-29 16:37:33,839][00126] Fps is (10 sec: 45875.8, 60 sec: 42598.4, 300 sec: 42043.0). Total num frames: 670089216. Throughput: 0: 42240.0. Samples: 552208420. Policy #0 lag: (min: 1.0, avg: 19.1, max: 42.0)
|
13995 |
+
[2024-03-29 16:37:33,840][00126] Avg episode reward: [(0, '0.537')]
|
13996 |
+
[2024-03-29 16:37:36,560][00497] Updated weights for policy 0, policy_version 40905 (0.0028)
|
13997 |
+
[2024-03-29 16:37:38,839][00126] Fps is (10 sec: 40960.0, 60 sec: 42325.4, 300 sec: 42043.0). Total num frames: 670269440. Throughput: 0: 42048.8. Samples: 552473100. Policy #0 lag: (min: 0.0, avg: 21.7, max: 41.0)
|
13998 |
+
[2024-03-29 16:37:38,842][00126] Avg episode reward: [(0, '0.545')]
|
13999 |
+
[2024-03-29 16:37:40,695][00497] Updated weights for policy 0, policy_version 40915 (0.0029)
|
14000 |
+
[2024-03-29 16:37:43,839][00126] Fps is (10 sec: 39321.7, 60 sec: 41779.2, 300 sec: 42098.6). Total num frames: 670482432. Throughput: 0: 42435.5. Samples: 552737520. Policy #0 lag: (min: 0.0, avg: 21.7, max: 41.0)
|
14001 |
+
[2024-03-29 16:37:43,840][00126] Avg episode reward: [(0, '0.531')]
|
14002 |
+
[2024-03-29 16:37:44,234][00497] Updated weights for policy 0, policy_version 40925 (0.0025)
|
14003 |
+
[2024-03-29 16:37:47,504][00497] Updated weights for policy 0, policy_version 40935 (0.0026)
|
14004 |
+
[2024-03-29 16:37:48,839][00126] Fps is (10 sec: 45875.5, 60 sec: 42598.4, 300 sec: 42043.0). Total num frames: 670728192. Throughput: 0: 42452.0. Samples: 552848560. Policy #0 lag: (min: 0.0, avg: 21.7, max: 41.0)
|
14005 |
+
[2024-03-29 16:37:48,840][00126] Avg episode reward: [(0, '0.477')]
|
14006 |
+
[2024-03-29 16:37:52,052][00497] Updated weights for policy 0, policy_version 40945 (0.0023)
|
14007 |
+
[2024-03-29 16:37:53,839][00126] Fps is (10 sec: 42598.3, 60 sec: 42325.3, 300 sec: 42043.0). Total num frames: 670908416. Throughput: 0: 42302.6. Samples: 553109700. Policy #0 lag: (min: 0.0, avg: 21.7, max: 41.0)
|
14008 |
+
[2024-03-29 16:37:53,840][00126] Avg episode reward: [(0, '0.490')]
|
14009 |
+
[2024-03-29 16:37:55,548][00476] Signal inference workers to stop experience collection... (19700 times)
|
14010 |
+
[2024-03-29 16:37:55,599][00497] InferenceWorker_p0-w0: stopping experience collection (19700 times)
|
14011 |
+
[2024-03-29 16:37:55,637][00476] Signal inference workers to resume experience collection... (19700 times)
|
14012 |
+
[2024-03-29 16:37:55,640][00497] InferenceWorker_p0-w0: resuming experience collection (19700 times)
|
14013 |
+
[2024-03-29 16:37:56,198][00497] Updated weights for policy 0, policy_version 40955 (0.0028)
|
14014 |
+
[2024-03-29 16:37:58,839][00126] Fps is (10 sec: 37683.1, 60 sec: 41779.2, 300 sec: 42043.0). Total num frames: 671105024. Throughput: 0: 42381.3. Samples: 553368500. Policy #0 lag: (min: 0.0, avg: 21.7, max: 41.0)
|
14015 |
+
[2024-03-29 16:37:58,840][00126] Avg episode reward: [(0, '0.538')]
|
14016 |
+
[2024-03-29 16:37:59,732][00497] Updated weights for policy 0, policy_version 40965 (0.0018)
|
14017 |
+
[2024-03-29 16:38:03,046][00497] Updated weights for policy 0, policy_version 40975 (0.0017)
|
14018 |
+
[2024-03-29 16:38:03,839][00126] Fps is (10 sec: 45875.3, 60 sec: 42598.4, 300 sec: 42154.1). Total num frames: 671367168. Throughput: 0: 42591.5. Samples: 553489160. Policy #0 lag: (min: 0.0, avg: 21.0, max: 42.0)
|
14019 |
+
[2024-03-29 16:38:03,840][00126] Avg episode reward: [(0, '0.427')]
|
14020 |
+
[2024-03-29 16:38:07,300][00497] Updated weights for policy 0, policy_version 40985 (0.0023)
|
14021 |
+
[2024-03-29 16:38:08,839][00126] Fps is (10 sec: 44236.7, 60 sec: 42598.4, 300 sec: 42098.5). Total num frames: 671547392. Throughput: 0: 42473.0. Samples: 553748440. Policy #0 lag: (min: 0.0, avg: 21.0, max: 42.0)
|
14022 |
+
[2024-03-29 16:38:08,840][00126] Avg episode reward: [(0, '0.562')]
|
14023 |
+
[2024-03-29 16:38:11,599][00497] Updated weights for policy 0, policy_version 40995 (0.0029)
|
14024 |
+
[2024-03-29 16:38:13,839][00126] Fps is (10 sec: 37683.0, 60 sec: 42052.2, 300 sec: 42098.6). Total num frames: 671744000. Throughput: 0: 42350.7. Samples: 554003180. Policy #0 lag: (min: 0.0, avg: 21.0, max: 42.0)
|
14025 |
+
[2024-03-29 16:38:13,840][00126] Avg episode reward: [(0, '0.380')]
|
14026 |
+
[2024-03-29 16:38:15,204][00497] Updated weights for policy 0, policy_version 41005 (0.0030)
|
14027 |
+
[2024-03-29 16:38:18,428][00497] Updated weights for policy 0, policy_version 41015 (0.0019)
|
14028 |
+
[2024-03-29 16:38:18,839][00126] Fps is (10 sec: 45875.0, 60 sec: 42598.3, 300 sec: 42209.6). Total num frames: 672006144. Throughput: 0: 42500.0. Samples: 554120920. Policy #0 lag: (min: 0.0, avg: 21.0, max: 42.0)
|
14029 |
+
[2024-03-29 16:38:18,840][00126] Avg episode reward: [(0, '0.561')]
|
14030 |
+
[2024-03-29 16:38:22,707][00497] Updated weights for policy 0, policy_version 41025 (0.0019)
|
14031 |
+
[2024-03-29 16:38:23,839][00126] Fps is (10 sec: 44237.1, 60 sec: 42598.5, 300 sec: 42154.1). Total num frames: 672186368. Throughput: 0: 42448.1. Samples: 554383260. Policy #0 lag: (min: 0.0, avg: 21.0, max: 42.0)
|
14032 |
+
[2024-03-29 16:38:23,840][00126] Avg episode reward: [(0, '0.552')]
|
14033 |
+
[2024-03-29 16:38:26,900][00497] Updated weights for policy 0, policy_version 41035 (0.0023)
|
14034 |
+
[2024-03-29 16:38:28,839][00126] Fps is (10 sec: 39321.5, 60 sec: 42325.3, 300 sec: 42154.1). Total num frames: 672399360. Throughput: 0: 42401.7. Samples: 554645600. Policy #0 lag: (min: 1.0, avg: 19.5, max: 42.0)
|
14035 |
+
[2024-03-29 16:38:28,840][00126] Avg episode reward: [(0, '0.606')]
|
14036 |
+
[2024-03-29 16:38:29,249][00476] Signal inference workers to stop experience collection... (19750 times)
|
14037 |
+
[2024-03-29 16:38:29,320][00497] InferenceWorker_p0-w0: stopping experience collection (19750 times)
|
14038 |
+
[2024-03-29 16:38:29,413][00476] Signal inference workers to resume experience collection... (19750 times)
|
14039 |
+
[2024-03-29 16:38:29,414][00497] InferenceWorker_p0-w0: resuming experience collection (19750 times)
|
14040 |
+
[2024-03-29 16:38:30,554][00497] Updated weights for policy 0, policy_version 41045 (0.0028)
|
14041 |
+
[2024-03-29 16:38:33,792][00497] Updated weights for policy 0, policy_version 41055 (0.0020)
|
14042 |
+
[2024-03-29 16:38:33,839][00126] Fps is (10 sec: 45874.6, 60 sec: 42598.3, 300 sec: 42209.6). Total num frames: 672645120. Throughput: 0: 42550.6. Samples: 554763340. Policy #0 lag: (min: 1.0, avg: 19.5, max: 42.0)
|
14043 |
+
[2024-03-29 16:38:33,840][00126] Avg episode reward: [(0, '0.567')]
|
14044 |
+
[2024-03-29 16:38:38,192][00497] Updated weights for policy 0, policy_version 41065 (0.0030)
|
14045 |
+
[2024-03-29 16:38:38,839][00126] Fps is (10 sec: 42598.4, 60 sec: 42598.4, 300 sec: 42209.6). Total num frames: 672825344. Throughput: 0: 42460.4. Samples: 555020420. Policy #0 lag: (min: 1.0, avg: 19.5, max: 42.0)
|
14046 |
+
[2024-03-29 16:38:38,840][00126] Avg episode reward: [(0, '0.562')]
|
14047 |
+
[2024-03-29 16:38:42,447][00497] Updated weights for policy 0, policy_version 41075 (0.0019)
|
14048 |
+
[2024-03-29 16:38:43,839][00126] Fps is (10 sec: 39322.1, 60 sec: 42598.4, 300 sec: 42209.6). Total num frames: 673038336. Throughput: 0: 42670.7. Samples: 555288680. Policy #0 lag: (min: 1.0, avg: 19.5, max: 42.0)
|
14049 |
+
[2024-03-29 16:38:43,840][00126] Avg episode reward: [(0, '0.507')]
|
14050 |
+
[2024-03-29 16:38:45,896][00497] Updated weights for policy 0, policy_version 41085 (0.0025)
|
14051 |
+
[2024-03-29 16:38:48,839][00126] Fps is (10 sec: 44236.8, 60 sec: 42325.3, 300 sec: 42265.2). Total num frames: 673267712. Throughput: 0: 42525.7. Samples: 555402820. Policy #0 lag: (min: 1.0, avg: 19.5, max: 42.0)
|
14052 |
+
[2024-03-29 16:38:48,840][00126] Avg episode reward: [(0, '0.547')]
|
14053 |
+
[2024-03-29 16:38:49,329][00497] Updated weights for policy 0, policy_version 41095 (0.0022)
|
14054 |
+
[2024-03-29 16:38:53,817][00497] Updated weights for policy 0, policy_version 41105 (0.0030)
|
14055 |
+
[2024-03-29 16:38:53,839][00126] Fps is (10 sec: 42598.0, 60 sec: 42598.4, 300 sec: 42209.6). Total num frames: 673464320. Throughput: 0: 42151.9. Samples: 555645280. Policy #0 lag: (min: 0.0, avg: 21.2, max: 41.0)
|
14056 |
+
[2024-03-29 16:38:53,840][00126] Avg episode reward: [(0, '0.548')]
|
14057 |
+
[2024-03-29 16:38:57,878][00497] Updated weights for policy 0, policy_version 41115 (0.0028)
|
14058 |
+
[2024-03-29 16:38:58,839][00126] Fps is (10 sec: 39321.7, 60 sec: 42598.4, 300 sec: 42209.6). Total num frames: 673660928. Throughput: 0: 42434.2. Samples: 555912720. Policy #0 lag: (min: 0.0, avg: 21.2, max: 41.0)
|
14059 |
+
[2024-03-29 16:38:58,840][00126] Avg episode reward: [(0, '0.486')]
|
14060 |
+
[2024-03-29 16:39:01,574][00497] Updated weights for policy 0, policy_version 41125 (0.0033)
|
14061 |
+
[2024-03-29 16:39:01,913][00476] Signal inference workers to stop experience collection... (19800 times)
|
14062 |
+
[2024-03-29 16:39:01,955][00497] InferenceWorker_p0-w0: stopping experience collection (19800 times)
|
14063 |
+
[2024-03-29 16:39:02,085][00476] Signal inference workers to resume experience collection... (19800 times)
|
14064 |
+
[2024-03-29 16:39:02,085][00497] InferenceWorker_p0-w0: resuming experience collection (19800 times)
|
14065 |
+
[2024-03-29 16:39:03,839][00126] Fps is (10 sec: 44237.1, 60 sec: 42325.3, 300 sec: 42265.2). Total num frames: 673906688. Throughput: 0: 42871.6. Samples: 556050140. Policy #0 lag: (min: 0.0, avg: 21.2, max: 41.0)
|
14066 |
+
[2024-03-29 16:39:03,840][00126] Avg episode reward: [(0, '0.471')]
|
14067 |
+
[2024-03-29 16:39:04,061][00476] Saving /workspace/metta/train_dir/b.a20.20x20_40x40.norm/checkpoint_p0/checkpoint_000041133_673923072.pth...
|
14068 |
+
[2024-03-29 16:39:04,396][00476] Removing /workspace/metta/train_dir/b.a20.20x20_40x40.norm/checkpoint_p0/checkpoint_000040513_663764992.pth
|
14069 |
+
[2024-03-29 16:39:04,973][00497] Updated weights for policy 0, policy_version 41135 (0.0023)
|
14070 |
+
[2024-03-29 16:39:08,839][00126] Fps is (10 sec: 42598.8, 60 sec: 42325.4, 300 sec: 42209.6). Total num frames: 674086912. Throughput: 0: 42110.7. Samples: 556278240. Policy #0 lag: (min: 0.0, avg: 21.2, max: 41.0)
|
14071 |
+
[2024-03-29 16:39:08,840][00126] Avg episode reward: [(0, '0.407')]
|
14072 |
+
[2024-03-29 16:39:09,202][00497] Updated weights for policy 0, policy_version 41145 (0.0020)
|
14073 |
+
[2024-03-29 16:39:13,317][00497] Updated weights for policy 0, policy_version 41155 (0.0018)
|
14074 |
+
[2024-03-29 16:39:13,839][00126] Fps is (10 sec: 40960.5, 60 sec: 42871.6, 300 sec: 42320.7). Total num frames: 674316288. Throughput: 0: 42402.4. Samples: 556553700. Policy #0 lag: (min: 0.0, avg: 21.2, max: 41.0)
|
14075 |
+
[2024-03-29 16:39:13,840][00126] Avg episode reward: [(0, '0.546')]
|
14076 |
+
[2024-03-29 16:39:17,051][00497] Updated weights for policy 0, policy_version 41165 (0.0026)
|
14077 |
+
[2024-03-29 16:39:18,839][00126] Fps is (10 sec: 44236.1, 60 sec: 42052.2, 300 sec: 42265.1). Total num frames: 674529280. Throughput: 0: 42638.2. Samples: 556682060. Policy #0 lag: (min: 0.0, avg: 21.2, max: 41.0)
|
14078 |
+
[2024-03-29 16:39:18,840][00126] Avg episode reward: [(0, '0.549')]
|
14079 |
+
[2024-03-29 16:39:20,298][00497] Updated weights for policy 0, policy_version 41175 (0.0019)
|
14080 |
+
[2024-03-29 16:39:23,839][00126] Fps is (10 sec: 40960.0, 60 sec: 42325.4, 300 sec: 42154.1). Total num frames: 674725888. Throughput: 0: 41927.3. Samples: 556907140. Policy #0 lag: (min: 0.0, avg: 22.4, max: 42.0)
|
14081 |
+
[2024-03-29 16:39:23,840][00126] Avg episode reward: [(0, '0.445')]
|
14082 |
+
[2024-03-29 16:39:24,951][00497] Updated weights for policy 0, policy_version 41185 (0.0025)
|
14083 |
+
[2024-03-29 16:39:28,839][00126] Fps is (10 sec: 39321.9, 60 sec: 42052.3, 300 sec: 42209.6). Total num frames: 674922496. Throughput: 0: 42118.6. Samples: 557184020. Policy #0 lag: (min: 0.0, avg: 22.4, max: 42.0)
|
14084 |
+
[2024-03-29 16:39:28,840][00126] Avg episode reward: [(0, '0.521')]
|
14085 |
+
[2024-03-29 16:39:28,863][00497] Updated weights for policy 0, policy_version 41195 (0.0021)
|
14086 |
+
[2024-03-29 16:39:32,756][00497] Updated weights for policy 0, policy_version 41205 (0.0036)
|
14087 |
+
[2024-03-29 16:39:33,839][00126] Fps is (10 sec: 42597.5, 60 sec: 41779.2, 300 sec: 42209.6). Total num frames: 675151872. Throughput: 0: 42405.8. Samples: 557311080. Policy #0 lag: (min: 0.0, avg: 22.4, max: 42.0)
|
14088 |
+
[2024-03-29 16:39:33,840][00126] Avg episode reward: [(0, '0.487')]
|
14089 |
+
[2024-03-29 16:39:35,981][00497] Updated weights for policy 0, policy_version 41215 (0.0021)
|
14090 |
+
[2024-03-29 16:39:37,084][00476] Signal inference workers to stop experience collection... (19850 times)
|
14091 |
+
[2024-03-29 16:39:37,107][00497] InferenceWorker_p0-w0: stopping experience collection (19850 times)
|
14092 |
+
[2024-03-29 16:39:37,301][00476] Signal inference workers to resume experience collection... (19850 times)
|
14093 |
+
[2024-03-29 16:39:37,301][00497] InferenceWorker_p0-w0: resuming experience collection (19850 times)
|
14094 |
+
[2024-03-29 16:39:38,839][00126] Fps is (10 sec: 45875.4, 60 sec: 42598.5, 300 sec: 42209.6). Total num frames: 675381248. Throughput: 0: 42253.0. Samples: 557546660. Policy #0 lag: (min: 0.0, avg: 22.4, max: 42.0)
|
14095 |
+
[2024-03-29 16:39:38,840][00126] Avg episode reward: [(0, '0.562')]
|
14096 |
+
[2024-03-29 16:39:40,591][00497] Updated weights for policy 0, policy_version 41225 (0.0025)
|
14097 |
+
[2024-03-29 16:39:43,839][00126] Fps is (10 sec: 39322.3, 60 sec: 41779.2, 300 sec: 42154.1). Total num frames: 675545088. Throughput: 0: 42018.8. Samples: 557803560. Policy #0 lag: (min: 0.0, avg: 22.4, max: 42.0)
|
14098 |
+
[2024-03-29 16:39:43,840][00126] Avg episode reward: [(0, '0.453')]
|
14099 |
+
[2024-03-29 16:39:44,775][00497] Updated weights for policy 0, policy_version 41235 (0.0029)
|
14100 |
+
[2024-03-29 16:39:48,516][00497] Updated weights for policy 0, policy_version 41245 (0.0019)
|
14101 |
+
[2024-03-29 16:39:48,839][00126] Fps is (10 sec: 39321.7, 60 sec: 41779.3, 300 sec: 42209.6). Total num frames: 675774464. Throughput: 0: 41765.8. Samples: 557929600. Policy #0 lag: (min: 0.0, avg: 19.5, max: 41.0)
|
14102 |
+
[2024-03-29 16:39:48,840][00126] Avg episode reward: [(0, '0.526')]
|
14103 |
+
[2024-03-29 16:39:51,924][00497] Updated weights for policy 0, policy_version 41255 (0.0021)
|
14104 |
+
[2024-03-29 16:39:53,839][00126] Fps is (10 sec: 45874.6, 60 sec: 42325.4, 300 sec: 42265.2). Total num frames: 676003840. Throughput: 0: 41891.0. Samples: 558163340. Policy #0 lag: (min: 0.0, avg: 19.5, max: 41.0)
|
14105 |
+
[2024-03-29 16:39:53,840][00126] Avg episode reward: [(0, '0.551')]
|
14106 |
+
[2024-03-29 16:39:56,178][00497] Updated weights for policy 0, policy_version 41265 (0.0029)
|
14107 |
+
[2024-03-29 16:39:58,839][00126] Fps is (10 sec: 39321.2, 60 sec: 41779.2, 300 sec: 42098.5). Total num frames: 676167680. Throughput: 0: 41679.4. Samples: 558429280. Policy #0 lag: (min: 0.0, avg: 19.5, max: 41.0)
|
14108 |
+
[2024-03-29 16:39:58,840][00126] Avg episode reward: [(0, '0.511')]
|
14109 |
+
[2024-03-29 16:40:00,250][00497] Updated weights for policy 0, policy_version 41275 (0.0024)
|
14110 |
+
[2024-03-29 16:40:03,839][00126] Fps is (10 sec: 39322.0, 60 sec: 41506.2, 300 sec: 42209.6). Total num frames: 676397056. Throughput: 0: 41672.2. Samples: 558557300. Policy #0 lag: (min: 0.0, avg: 19.5, max: 41.0)
|
14111 |
+
[2024-03-29 16:40:03,840][00126] Avg episode reward: [(0, '0.490')]
|
14112 |
+
[2024-03-29 16:40:03,971][00497] Updated weights for policy 0, policy_version 41285 (0.0018)
|
14113 |
+
[2024-03-29 16:40:07,296][00497] Updated weights for policy 0, policy_version 41295 (0.0025)
|
14114 |
+
[2024-03-29 16:40:08,839][00126] Fps is (10 sec: 47514.1, 60 sec: 42598.4, 300 sec: 42265.2). Total num frames: 676642816. Throughput: 0: 42204.4. Samples: 558806340. Policy #0 lag: (min: 0.0, avg: 19.5, max: 41.0)
|
14115 |
+
[2024-03-29 16:40:08,840][00126] Avg episode reward: [(0, '0.494')]
|
14116 |
+
[2024-03-29 16:40:11,305][00476] Signal inference workers to stop experience collection... (19900 times)
|
14117 |
+
[2024-03-29 16:40:11,306][00476] Signal inference workers to resume experience collection... (19900 times)
|
14118 |
+
[2024-03-29 16:40:11,333][00497] InferenceWorker_p0-w0: stopping experience collection (19900 times)
|
14119 |
+
[2024-03-29 16:40:11,355][00497] InferenceWorker_p0-w0: resuming experience collection (19900 times)
|
14120 |
+
[2024-03-29 16:40:11,611][00497] Updated weights for policy 0, policy_version 41305 (0.0037)
|
14121 |
+
[2024-03-29 16:40:13,839][00126] Fps is (10 sec: 40959.9, 60 sec: 41506.1, 300 sec: 42154.1). Total num frames: 676806656. Throughput: 0: 41582.7. Samples: 559055240. Policy #0 lag: (min: 0.0, avg: 21.0, max: 42.0)
|
14122 |
+
[2024-03-29 16:40:13,840][00126] Avg episode reward: [(0, '0.530')]
|
14123 |
+
[2024-03-29 16:40:15,704][00497] Updated weights for policy 0, policy_version 41315 (0.0030)
|
14124 |
+
[2024-03-29 16:40:18,839][00126] Fps is (10 sec: 37683.2, 60 sec: 41506.3, 300 sec: 42154.1). Total num frames: 677019648. Throughput: 0: 41593.9. Samples: 559182800. Policy #0 lag: (min: 0.0, avg: 21.0, max: 42.0)
|
14125 |
+
[2024-03-29 16:40:18,841][00126] Avg episode reward: [(0, '0.461')]
|
14126 |
+
[2024-03-29 16:40:19,787][00497] Updated weights for policy 0, policy_version 41325 (0.0027)
|
14127 |
+
[2024-03-29 16:40:23,011][00497] Updated weights for policy 0, policy_version 41335 (0.0020)
|
14128 |
+
[2024-03-29 16:40:23,839][00126] Fps is (10 sec: 45874.7, 60 sec: 42325.2, 300 sec: 42154.1). Total num frames: 677265408. Throughput: 0: 41884.8. Samples: 559431480. Policy #0 lag: (min: 0.0, avg: 21.0, max: 42.0)
|
14129 |
+
[2024-03-29 16:40:23,840][00126] Avg episode reward: [(0, '0.522')]
|
14130 |
+
[2024-03-29 16:40:27,212][00497] Updated weights for policy 0, policy_version 41345 (0.0025)
|