Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +6 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/argv.txt +40 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/model.safetensors +3 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/optimizer.bin +3 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_0.pkl +3 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/scheduler.bin +3 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/envs.txt +17 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/run_config.json +310 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug-internal.log +0 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug.log +25 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/config.yaml +448 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/output.log +203 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/requirements.txt +199 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/wandb-metadata.json +132 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/wandb-summary.json +1 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug-internal.log +0 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug.log +25 -0
- act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/run-agq65opl.wandb +3 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/argv.txt +40 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/model.safetensors +3 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/optimizer.bin +3 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_0.pkl +3 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_1.pkl +3 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_2.pkl +3 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/scheduler.bin +3 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/envs.txt +17 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/run_config.json +310 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/debug-internal.log +0 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/debug.log +25 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/config.yaml +448 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/output.log +179 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/requirements.txt +219 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/wandb-metadata.json +132 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/wandb-summary.json +1 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-core.log +19 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-internal.log +0 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug.log +25 -0
- act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/run-26s2ml9t.wandb +3 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/argv.txt +40 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/model.safetensors +3 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/optimizer.bin +3 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/random_states_0.pkl +3 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/scheduler.bin +3 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/envs.txt +17 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/run_config.json +310 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/debug-internal.log +0 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/debug.log +25 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/config.yaml +448 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/output.log +237 -0
- act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/requirements.txt +219 -0
.gitattributes
CHANGED
|
@@ -2100,3 +2100,9 @@ dp/diffusion-policy-g1-real/dp-remove-the-cap-turn-on-the-faucet-and-fill-the-bo
|
|
| 2100 |
dp/diffusion-policy-g1-real/dp-rotate-to-pour-ham-into-plate-and-push-the-cart-forward.g1-real.cosin.lr1.0e-04.b128.gpus4.2601240523/wandb/run-20260124_052421-2gdpwmjh/run-2gdpwmjh.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2101 |
dp/diffusion-policy-g1-real/dp-spray-the-bowl-and-wipe-it-and-stack-it-up.g1-real.cosin.lr1.0e-04.b128.gpus4.2601240526/wandb/run-20260124_052721-45k2nxui/run-45k2nxui.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2102 |
egovla/simple/G1WholebodyXMovePick-v0/trainer_state.json filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2100 |
dp/diffusion-policy-g1-real/dp-rotate-to-pour-ham-into-plate-and-push-the-cart-forward.g1-real.cosin.lr1.0e-04.b128.gpus4.2601240523/wandb/run-20260124_052421-2gdpwmjh/run-2gdpwmjh.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2101 |
dp/diffusion-policy-g1-real/dp-spray-the-bowl-and-wipe-it-and-stack-it-up.g1-real.cosin.lr1.0e-04.b128.gpus4.2601240526/wandb/run-20260124_052721-45k2nxui/run-45k2nxui.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2102 |
egovla/simple/G1WholebodyXMovePick-v0/trainer_state.json filter=lfs diff=lfs merge=lfs -text
|
| 2103 |
+
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/run-agq65opl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2104 |
+
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/run-26s2ml9t.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2105 |
+
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/run-ii0lixdx.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2106 |
+
act/act-g1-sim/g1wholebodytabletopgrasp-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603190700/wandb/run-20260319_070047-kgnkklsw/run-kgnkklsw.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2107 |
+
act/act-g1-sim/g1wholebodyxmovebendpickteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604091356/wandb/run-20260409_135628-skni4cjp/run-skni4cjp.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2108 |
+
act/act-g1-sim/g1wholebodyxmovepick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604040237/wandb/run-20260404_023728-qq4fojro/run-qq4fojro.wandb filter=lfs diff=lfs merge=lfs -text
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/argv.txt
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
scripts/train.py
|
| 2 |
+
simple_act_config
|
| 3 |
+
--seed=2026
|
| 4 |
+
--exp=g1wholebodybendpick-v0
|
| 5 |
+
--train.name=act-g1
|
| 6 |
+
--log.report-to=wandb
|
| 7 |
+
--train.data_parallel=ddp
|
| 8 |
+
--train.mixed_precision=bf16
|
| 9 |
+
--train.train-batch-size=32
|
| 10 |
+
--train.warmup-steps=1000
|
| 11 |
+
--train.warmup-ratio=None
|
| 12 |
+
--train.checkpointing-steps=5000
|
| 13 |
+
--train.validation_steps=500
|
| 14 |
+
--train.val_num_batches=20
|
| 15 |
+
--train.gradient_accumulation_steps=1
|
| 16 |
+
--train.max-training-steps=40000
|
| 17 |
+
--train.learning-rate=1e-4
|
| 18 |
+
--train.max-grad-norm=1.0
|
| 19 |
+
--train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 20 |
+
--train.lr_scheduler_kwargs.betas 0.95 0.999
|
| 21 |
+
--train.lr_scheduler_type=cosine
|
| 22 |
+
--data.root_dir=/data/jliu/data
|
| 23 |
+
--data.train-repo-ids=G1WholebodyBendPick-v0
|
| 24 |
+
--data.transform.repack.action_chunk_size=30
|
| 25 |
+
--data.transform.repack.pad-action-dim=36
|
| 26 |
+
--data.transform.repack.pad-state-dim=36
|
| 27 |
+
--data.transform.field.stat-path=meta/stats_psi0.json
|
| 28 |
+
--data.transform.field.stat-action-key=action
|
| 29 |
+
--data.transform.field.stat-state-key=states
|
| 30 |
+
--data.transform.field.normalize-state
|
| 31 |
+
--data.transform.field.action-norm-type=bounds
|
| 32 |
+
--data.transform.field.pad-action-dim=36
|
| 33 |
+
--data.transform.field.pad-state-dim=36
|
| 34 |
+
--data.transform.model.img-aug
|
| 35 |
+
--model.chunk-size=30
|
| 36 |
+
--model.n-action-steps=30
|
| 37 |
+
--model.action-dim=36
|
| 38 |
+
--model.state-dim=36
|
| 39 |
+
--model.use-vae
|
| 40 |
+
--model.kl-weight=10.0
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fb015062b468f83f0f8d65b9e097353c9e1501e127f17ec8a509150866d178e7
|
| 3 |
+
size 206658952
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/optimizer.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9e333a033a30c0ed624973e1e81ae16152f89080f9d33944ed80782e04945056
|
| 3 |
+
size 413111371
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_0.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8ce2a0461b67075096482bb9b83ea22fa0f07f2d3b9e6640249cdb0e3ddd117d
|
| 3 |
+
size 15473
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/scheduler.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:edd2ef4de5774723b53b76baced1d134ab2a534e3951c7a3ebfbc157468c8f72
|
| 3 |
+
size 1401
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/envs.txt
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
OMP_NUM_THREADS=32
|
| 2 |
+
HF_HOME=/data/cache
|
| 3 |
+
TORCH_HOME=/data/cache
|
| 4 |
+
HF_TOKEN=hf_...TiKa
|
| 5 |
+
HF_LEROBOT_HOME=/data/data/lerobot
|
| 6 |
+
WE_HOME=Not Set
|
| 7 |
+
DATA_HOME=/data/data
|
| 8 |
+
UV_CACHE_DIR=/data/cache
|
| 9 |
+
WANDB_API_KEY=90e...5c06
|
| 10 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
|
| 11 |
+
CUDA_VISIBLE_DEVICES=0,1,2,3
|
| 12 |
+
WORLD_SIZE=4
|
| 13 |
+
LOCAL_WORLD_SIZE=4
|
| 14 |
+
RANK=0
|
| 15 |
+
LOCAL_RANK=0
|
| 16 |
+
MASTER_ADDR=nebula99
|
| 17 |
+
MASTER_PORT=37919
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/run_config.json
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"exp": "g1wholebodybendpick-v0",
|
| 3 |
+
"seed": 2026,
|
| 4 |
+
"auto_tag_run": false,
|
| 5 |
+
"eval": false,
|
| 6 |
+
"debug": false,
|
| 7 |
+
"timestamp": "2603181426",
|
| 8 |
+
"log": {
|
| 9 |
+
"logging_dir": "logs",
|
| 10 |
+
"report_to": "wandb",
|
| 11 |
+
"log_freq": 100
|
| 12 |
+
},
|
| 13 |
+
"wandb": {
|
| 14 |
+
"project": "psi",
|
| 15 |
+
"entity": "jliu530-soochow-university",
|
| 16 |
+
"group": "act-g1",
|
| 17 |
+
"id": "agq65opl",
|
| 18 |
+
"name": "g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426",
|
| 19 |
+
"resume": "allow"
|
| 20 |
+
},
|
| 21 |
+
"train": {
|
| 22 |
+
"num_workers": 8,
|
| 23 |
+
"overfit_single_batch": false,
|
| 24 |
+
"name": "act-g1",
|
| 25 |
+
"resume_from_checkpoint": null,
|
| 26 |
+
"skip_resumed_steps": false,
|
| 27 |
+
"hf_token": ".hf_token",
|
| 28 |
+
"lora": false,
|
| 29 |
+
"output_dir": ".runs",
|
| 30 |
+
"gradient_accumulation_steps": 1,
|
| 31 |
+
"mixed_precision": "bf16",
|
| 32 |
+
"max_grad_norm": 1.0,
|
| 33 |
+
"optimizer_foreach": null,
|
| 34 |
+
"train_batch_size": 32,
|
| 35 |
+
"val_batch_size": 16,
|
| 36 |
+
"val_num_batches": 20,
|
| 37 |
+
"checkpointing_steps": 5000,
|
| 38 |
+
"max_checkpoints_to_keep": null,
|
| 39 |
+
"validation_steps": 500,
|
| 40 |
+
"learning_rate": 0.0001,
|
| 41 |
+
"lr_scheduler_type": "cosine",
|
| 42 |
+
"lr_scheduler_kwargs": {
|
| 43 |
+
"betas": [
|
| 44 |
+
0.95,
|
| 45 |
+
0.999
|
| 46 |
+
],
|
| 47 |
+
"weight_decay": 1e-6,
|
| 48 |
+
"eps": 1e-8
|
| 49 |
+
},
|
| 50 |
+
"scheduler_specific_kwargs": {},
|
| 51 |
+
"data_parallel": "ddp",
|
| 52 |
+
"sharding_strategy": "full-shard",
|
| 53 |
+
"deepspeed_config": "/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json",
|
| 54 |
+
"enable_gradient_checkpointing": true,
|
| 55 |
+
"enable_mixed_precision_training": true,
|
| 56 |
+
"reduce_in_full_precision": true,
|
| 57 |
+
"max_training_steps": 40000,
|
| 58 |
+
"num_train_epochs": null,
|
| 59 |
+
"warmup_steps": 1000,
|
| 60 |
+
"warmup_ratio": null
|
| 61 |
+
},
|
| 62 |
+
"data": {
|
| 63 |
+
"transform": {
|
| 64 |
+
"repack": {
|
| 65 |
+
"dataset_name": "simple",
|
| 66 |
+
"num_past_frames": 0,
|
| 67 |
+
"action_chunk_size": 30,
|
| 68 |
+
"pad_action_dim": 36,
|
| 69 |
+
"pad_state_dim": 36
|
| 70 |
+
},
|
| 71 |
+
"model": {
|
| 72 |
+
"resize": {
|
| 73 |
+
"size": [
|
| 74 |
+
256,
|
| 75 |
+
480
|
| 76 |
+
]
|
| 77 |
+
},
|
| 78 |
+
"center_crop": {
|
| 79 |
+
"size": [
|
| 80 |
+
224,
|
| 81 |
+
224
|
| 82 |
+
]
|
| 83 |
+
},
|
| 84 |
+
"color_jitter": {
|
| 85 |
+
"brightness": 0.2,
|
| 86 |
+
"contrast": [
|
| 87 |
+
0.8,
|
| 88 |
+
1.2
|
| 89 |
+
],
|
| 90 |
+
"saturation": [
|
| 91 |
+
0.8,
|
| 92 |
+
1.2
|
| 93 |
+
],
|
| 94 |
+
"hue": 0.05
|
| 95 |
+
},
|
| 96 |
+
"normalize": {
|
| 97 |
+
"mean": [
|
| 98 |
+
0.485,
|
| 99 |
+
0.456,
|
| 100 |
+
0.406
|
| 101 |
+
],
|
| 102 |
+
"std": [
|
| 103 |
+
0.229,
|
| 104 |
+
0.224,
|
| 105 |
+
0.225
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
"img_aug": true
|
| 109 |
+
},
|
| 110 |
+
"field": {
|
| 111 |
+
"stat_path": "meta/stats_psi0.json",
|
| 112 |
+
"action_norm_type": "bounds",
|
| 113 |
+
"stat_action_key": "action",
|
| 114 |
+
"stat_state_key": "states",
|
| 115 |
+
"use_norm_mask": false,
|
| 116 |
+
"action_norm_masks": [
|
| 117 |
+
true,
|
| 118 |
+
true,
|
| 119 |
+
true,
|
| 120 |
+
true,
|
| 121 |
+
true,
|
| 122 |
+
true,
|
| 123 |
+
false
|
| 124 |
+
],
|
| 125 |
+
"action_min": [
|
| 126 |
+
-0.13059291243553162,
|
| 127 |
+
-0.09108058363199234,
|
| 128 |
+
-0.0024844733998179436,
|
| 129 |
+
-0.20733775198459625,
|
| 130 |
+
-0.15850023925304413,
|
| 131 |
+
-0.17450474202632904,
|
| 132 |
+
-0.2997315526008606,
|
| 133 |
+
-0.015391111373901367,
|
| 134 |
+
-0.34571564197540283,
|
| 135 |
+
-0.4991437792778015,
|
| 136 |
+
0.0,
|
| 137 |
+
0.0,
|
| 138 |
+
0.0,
|
| 139 |
+
0.0,
|
| 140 |
+
-0.1015840545296669,
|
| 141 |
+
-0.06647031009197235,
|
| 142 |
+
-0.16578954458236694,
|
| 143 |
+
-0.14477218687534332,
|
| 144 |
+
-0.3665394186973572,
|
| 145 |
+
-0.28364259004592896,
|
| 146 |
+
-0.1775387078523636,
|
| 147 |
+
-0.48419490456581116,
|
| 148 |
+
-0.7551082968711853,
|
| 149 |
+
-0.2692946195602417,
|
| 150 |
+
-0.03164339065551758,
|
| 151 |
+
-0.00003876500704791397,
|
| 152 |
+
-0.3909206688404083,
|
| 153 |
+
0.0,
|
| 154 |
+
-0.04351663216948509,
|
| 155 |
+
-0.014203650876879692,
|
| 156 |
+
-0.049649015069007874,
|
| 157 |
+
0.44999998807907104,
|
| 158 |
+
0.0,
|
| 159 |
+
0.0,
|
| 160 |
+
0.0,
|
| 161 |
+
0.0
|
| 162 |
+
],
|
| 163 |
+
"action_max": [
|
| 164 |
+
0.08620641380548477,
|
| 165 |
+
0.13058121502399445,
|
| 166 |
+
0.22948147356510162,
|
| 167 |
+
0.020551620051264763,
|
| 168 |
+
0.005824880674481392,
|
| 169 |
+
0.010019193403422832,
|
| 170 |
+
8.43817247186962e-7,
|
| 171 |
+
0.39566752314567566,
|
| 172 |
+
0.0,
|
| 173 |
+
0.0,
|
| 174 |
+
0.4860266447067261,
|
| 175 |
+
1.0467392206192017,
|
| 176 |
+
0.6470075845718384,
|
| 177 |
+
0.8298009037971497,
|
| 178 |
+
0.03516175225377083,
|
| 179 |
+
0.11019192636013031,
|
| 180 |
+
0.04779902100563049,
|
| 181 |
+
0.12850724160671234,
|
| 182 |
+
0.000038688118365826085,
|
| 183 |
+
0.0012142359046265483,
|
| 184 |
+
0.000033343669201713055,
|
| 185 |
+
0.002679983852431178,
|
| 186 |
+
0.00041063950629904866,
|
| 187 |
+
0.1973484456539154,
|
| 188 |
+
0.2633756697177887,
|
| 189 |
+
0.34943076968193054,
|
| 190 |
+
0.0012102096807211637,
|
| 191 |
+
0.8342975974082947,
|
| 192 |
+
0.31870752573013306,
|
| 193 |
+
0.45533719658851624,
|
| 194 |
+
0.15729404985904694,
|
| 195 |
+
0.75,
|
| 196 |
+
0.0,
|
| 197 |
+
0.0,
|
| 198 |
+
0.0,
|
| 199 |
+
0.0
|
| 200 |
+
],
|
| 201 |
+
"state_min": [
|
| 202 |
+
-0.13899999856948853,
|
| 203 |
+
-0.09099991619586945,
|
| 204 |
+
-5.989517215532203e-11,
|
| 205 |
+
-0.20900000631809235,
|
| 206 |
+
-0.1589999943971634,
|
| 207 |
+
-0.210999995470047,
|
| 208 |
+
-0.3009999990463257,
|
| 209 |
+
-0.01600000075995922,
|
| 210 |
+
-0.1860000044107437,
|
| 211 |
+
-0.6940000057220459,
|
| 212 |
+
0.0,
|
| 213 |
+
0.0,
|
| 214 |
+
0.0,
|
| 215 |
+
0.0,
|
| 216 |
+
-0.1019991859793663,
|
| 217 |
+
-0.06899992376565933,
|
| 218 |
+
-0.16899999976158142,
|
| 219 |
+
-0.14499999582767487,
|
| 220 |
+
-0.3709999918937683,
|
| 221 |
+
-0.28700000047683716,
|
| 222 |
+
-0.17800045013427734,
|
| 223 |
+
-0.4869999885559082,
|
| 224 |
+
-0.7599999904632568,
|
| 225 |
+
-0.27300000190734863,
|
| 226 |
+
-0.029999999329447746,
|
| 227 |
+
0.0,
|
| 228 |
+
-0.39100033044815063,
|
| 229 |
+
-0.0010000000474974513,
|
| 230 |
+
0.0,
|
| 231 |
+
-0.15000000596046448,
|
| 232 |
+
0.0,
|
| 233 |
+
0.44999998807907104,
|
| 234 |
+
0.0,
|
| 235 |
+
0.0,
|
| 236 |
+
0.0,
|
| 237 |
+
0.0
|
| 238 |
+
],
|
| 239 |
+
"state_max": [
|
| 240 |
+
0.0860000029206276,
|
| 241 |
+
0.2720000147819519,
|
| 242 |
+
0.23100000619888306,
|
| 243 |
+
0.0,
|
| 244 |
+
6.510182259944486e-8,
|
| 245 |
+
0.0,
|
| 246 |
+
0.0,
|
| 247 |
+
0.5550000071525574,
|
| 248 |
+
0.02100004442036152,
|
| 249 |
+
0.0,
|
| 250 |
+
0.5429999828338623,
|
| 251 |
+
1.13100004196167,
|
| 252 |
+
0.5770000219345093,
|
| 253 |
+
0.9580000042915344,
|
| 254 |
+
0.07141251862049103,
|
| 255 |
+
0.10899999737739563,
|
| 256 |
+
0.04699999839067459,
|
| 257 |
+
0.13600000739097595,
|
| 258 |
+
0.0,
|
| 259 |
+
0.003000000026077032,
|
| 260 |
+
0.0,
|
| 261 |
+
0.009999999776482582,
|
| 262 |
+
0.0020000000949949026,
|
| 263 |
+
0.2029999941587448,
|
| 264 |
+
0.2759999930858612,
|
| 265 |
+
0.3499999940395355,
|
| 266 |
+
0.003000000026077032,
|
| 267 |
+
0.8370000123977661,
|
| 268 |
+
0.0,
|
| 269 |
+
0.0,
|
| 270 |
+
0.0,
|
| 271 |
+
0.75,
|
| 272 |
+
0.0,
|
| 273 |
+
0.0,
|
| 274 |
+
0.0,
|
| 275 |
+
0.0
|
| 276 |
+
],
|
| 277 |
+
"normalize_state": true,
|
| 278 |
+
"pad_action_dim": 36,
|
| 279 |
+
"pad_state_dim": 36
|
| 280 |
+
}
|
| 281 |
+
},
|
| 282 |
+
"root_dir": "/data/jliu/data",
|
| 283 |
+
"train_repo_ids": [
|
| 284 |
+
"G1WholebodyBendPick-v0"
|
| 285 |
+
],
|
| 286 |
+
"val_repo_ids": [
|
| 287 |
+
"G1WholebodyBendPick-v0"
|
| 288 |
+
]
|
| 289 |
+
},
|
| 290 |
+
"model": {
|
| 291 |
+
"n_obs_steps": 1,
|
| 292 |
+
"chunk_size": 30,
|
| 293 |
+
"n_action_steps": 30,
|
| 294 |
+
"action_dim": 36,
|
| 295 |
+
"state_dim": 36,
|
| 296 |
+
"dim_model": 512,
|
| 297 |
+
"n_heads": 8,
|
| 298 |
+
"dim_feedforward": 3200,
|
| 299 |
+
"feedforward_activation": "relu",
|
| 300 |
+
"n_encoder_layers": 4,
|
| 301 |
+
"n_decoder_layers": 1,
|
| 302 |
+
"pre_norm": false,
|
| 303 |
+
"dropout": 0.1,
|
| 304 |
+
"use_vae": true,
|
| 305 |
+
"latent_dim": 32,
|
| 306 |
+
"n_vae_encoder_layers": 4,
|
| 307 |
+
"kl_weight": 10.0,
|
| 308 |
+
"temporal_ensemble_coeff": null
|
| 309 |
+
}
|
| 310 |
+
}
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-03-18 14:26:46,542 INFO MainThread:280442 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-03-18 14:26:46,542 INFO MainThread:280442 [wandb_setup.py:_flush():81] Configure stats pid to 280442
|
| 3 |
+
2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug.log
|
| 5 |
+
2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug-internal.log
|
| 6 |
+
2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-03-18 14:26:46,802 INFO MainThread:280442 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-03-18 14:26:46,814 INFO MainThread:280442 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-03-18 14:26:46,815 INFO MainThread:280442 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-03-18 14:26:46,822 INFO MainThread:280442 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-03-18 14:26:48,282 INFO MainThread:280442 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-03-18 14:26:48,444 INFO MainThread:280442 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-03-18 14:26:48,451 INFO MainThread:280442 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-03-18 14:26:48,454 INFO MainThread:280442 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodybendpick-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2603181426', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.13059291243553162, -0.09108058363199234, -0.0024844733998179436, -0.20733775198459625, -0.15850023925304413, -0.17450474202632904, -0.2997315526008606, -0.015391111373901367, -0.34571564197540283, -0.4991437792778015, 0.0, 0.0, 0.0, 0.0, -0.1015840545296669, -0.06647031009197235, -0.16578954458236694, -0.14477218687534332, -0.3665394186973572, -0.28364259004592896, -0.1775387078523636, -0.48419490456581116, -0.7551082968711853, -0.2692946195602417, -0.03164339065551758, -3.876500704791397e-05, -0.3909206688404083, 0.0, -0.04351663216948509, -0.014203650876879692, -0.049649015069007874, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.08620641380548477, 0.13058121502399445, 0.22948147356510162, 0.020551620051264763, 0.005824880674481392, 0.010019193403422832, 8.43817247186962e-07, 0.39566752314567566, 0.0, 0.0, 0.4860266447067261, 1.0467392206192017, 0.6470075845718384, 0.8298009037971497, 0.03516175225377083, 0.11019192636013031, 0.04779902100563049, 0.12850724160671234, 3.8688118365826085e-05, 0.0012142359046265483, 3.3343669201713055e-05, 0.002679983852431178, 0.00041063950629904866, 0.1973484456539154, 0.2633756697177887, 0.34943076968193054, 0.0012102096807211637, 0.8342975974082947, 0.31870752573013306, 0.45533719658851624, 0.15729404985904694, 0.75, 0.0, 0.0, 0.0, 0.0], 'state_min': [-0.13899999856948853, -0.09099991619586945, -5.989517215532203e-11, -0.20900000631809235, -0.1589999943971634, -0.210999995470047, -0.3009999990463257, -0.01600000075995922, -0.1860000044107437, -0.6940000057220459, 0.0, 0.0, 0.0, 0.0, -0.1019991859793663, -0.06899992376565933, -0.16899999976158142, -0.14499999582767487, -0.3709999918937683, -0.28700000047683716, -0.17800045013427734, -0.4869999885559082, -0.7599999904632568, -0.27300000190734863, -0.029999999329447746, 0.0, -0.39100033044815063, -0.0010000000474974513, 0.0, -0.15000000596046448, 0.0, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.0860000029206276, 0.2720000147819519, 0.23100000619888306, 0.0, 6.510182259944486e-08, 0.0, 0.0, 0.5550000071525574, 0.02100004442036152, 0.0, 0.5429999828338623, 1.13100004196167, 0.5770000219345093, 0.9580000042915344, 0.07141251862049103, 0.10899999737739563, 0.04699999839067459, 0.13600000739097595, 0.0, 0.003000000026077032, 0.0, 0.009999999776482582, 0.0020000000949949026, 0.2029999941587448, 0.2759999930858612, 0.3499999940395355, 0.003000000026077032, 0.8370000123977661, 0.0, 0.0, 0.0, 0.75, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyBendPick-v0'], 'val_repo_ids': ['G1WholebodyBendPick-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '0,1,2,3', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '37919'}}
|
| 21 |
+
2026-03-18 19:53:48,477 INFO MainThread:280442 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/agq65opl
|
| 22 |
+
2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
|
| 23 |
+
2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_restore():2472] restore
|
| 24 |
+
2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_restore():2478] restore done
|
| 25 |
+
2026-03-18 19:53:50,040 INFO MainThread:280442 [wandb_run.py:_footer_sync_info():3868] logging synced files
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/config.yaml
ADDED
|
@@ -0,0 +1,448 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.1
|
| 4 |
+
e:
|
| 5 |
+
0fepodeqeh00upnrzhqar4dfcf23o49l:
|
| 6 |
+
args:
|
| 7 |
+
- simple_act_config
|
| 8 |
+
- --seed=2026
|
| 9 |
+
- --exp=g1wholebodybendpick-v0
|
| 10 |
+
- --train.name=act-g1
|
| 11 |
+
- --log.report-to=wandb
|
| 12 |
+
- --train.data_parallel=ddp
|
| 13 |
+
- --train.mixed_precision=bf16
|
| 14 |
+
- --train.train-batch-size=32
|
| 15 |
+
- --train.warmup-steps=1000
|
| 16 |
+
- --train.warmup-ratio=None
|
| 17 |
+
- --train.checkpointing-steps=5000
|
| 18 |
+
- --train.validation_steps=500
|
| 19 |
+
- --train.val_num_batches=20
|
| 20 |
+
- --train.gradient_accumulation_steps=1
|
| 21 |
+
- --train.max-training-steps=40000
|
| 22 |
+
- --train.learning-rate=1e-4
|
| 23 |
+
- --train.max-grad-norm=1.0
|
| 24 |
+
- --train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 25 |
+
- --train.lr_scheduler_kwargs.betas
|
| 26 |
+
- "0.95"
|
| 27 |
+
- "0.999"
|
| 28 |
+
- --train.lr_scheduler_type=cosine
|
| 29 |
+
- --data.root_dir=/data/jliu/data
|
| 30 |
+
- --data.train-repo-ids=G1WholebodyBendPick-v0
|
| 31 |
+
- --data.transform.repack.action_chunk_size=30
|
| 32 |
+
- --data.transform.repack.pad-action-dim=36
|
| 33 |
+
- --data.transform.repack.pad-state-dim=36
|
| 34 |
+
- --data.transform.field.stat-path=meta/stats_psi0.json
|
| 35 |
+
- --data.transform.field.stat-action-key=action
|
| 36 |
+
- --data.transform.field.stat-state-key=states
|
| 37 |
+
- --data.transform.field.normalize-state
|
| 38 |
+
- --data.transform.field.action-norm-type=bounds
|
| 39 |
+
- --data.transform.field.pad-action-dim=36
|
| 40 |
+
- --data.transform.field.pad-state-dim=36
|
| 41 |
+
- --data.transform.model.img-aug
|
| 42 |
+
- --model.chunk-size=30
|
| 43 |
+
- --model.n-action-steps=30
|
| 44 |
+
- --model.action-dim=36
|
| 45 |
+
- --model.state-dim=36
|
| 46 |
+
- --model.use-vae
|
| 47 |
+
- --model.kl-weight=10.0
|
| 48 |
+
codePath: scripts/train.py
|
| 49 |
+
codePathLocal: scripts/train.py
|
| 50 |
+
cpu_count: 128
|
| 51 |
+
cpu_count_logical: 128
|
| 52 |
+
cudaVersion: "12.9"
|
| 53 |
+
disk:
|
| 54 |
+
/:
|
| 55 |
+
total: "105089261568"
|
| 56 |
+
used: "99538649088"
|
| 57 |
+
email: jliu530@163.com
|
| 58 |
+
executable: /data/jliu/psi/.venv-psi/bin/python3
|
| 59 |
+
git:
|
| 60 |
+
commit: 04614628ecb677f5f278e2d31c1103aed8127f26
|
| 61 |
+
remote: https://github.com/songlin/psi.git
|
| 62 |
+
gpu: NVIDIA A100-SXM4-80GB
|
| 63 |
+
gpu_count: 8
|
| 64 |
+
gpu_nvidia:
|
| 65 |
+
- architecture: Ampere
|
| 66 |
+
cudaCores: 6912
|
| 67 |
+
memoryTotal: "85899345920"
|
| 68 |
+
name: NVIDIA A100-SXM4-80GB
|
| 69 |
+
uuid: GPU-37370f00-90f5-4e03-f763-7a35649e6783
|
| 70 |
+
- architecture: Ampere
|
| 71 |
+
cudaCores: 6912
|
| 72 |
+
memoryTotal: "85899345920"
|
| 73 |
+
name: NVIDIA A100-SXM4-80GB
|
| 74 |
+
uuid: GPU-d4b40383-188b-cc16-9180-20c1a71a777f
|
| 75 |
+
- architecture: Ampere
|
| 76 |
+
cudaCores: 6912
|
| 77 |
+
memoryTotal: "85899345920"
|
| 78 |
+
name: NVIDIA A100-SXM4-80GB
|
| 79 |
+
uuid: GPU-0f3d827c-66cf-04da-b182-4ed0414a2549
|
| 80 |
+
- architecture: Ampere
|
| 81 |
+
cudaCores: 6912
|
| 82 |
+
memoryTotal: "85899345920"
|
| 83 |
+
name: NVIDIA A100-SXM4-80GB
|
| 84 |
+
uuid: GPU-164a49bb-43d3-2250-59e2-1b40eee0757f
|
| 85 |
+
- architecture: Ampere
|
| 86 |
+
cudaCores: 6912
|
| 87 |
+
memoryTotal: "85899345920"
|
| 88 |
+
name: NVIDIA A100-SXM4-80GB
|
| 89 |
+
uuid: GPU-c0471f40-ae19-5371-a1af-da406ca30f83
|
| 90 |
+
- architecture: Ampere
|
| 91 |
+
cudaCores: 6912
|
| 92 |
+
memoryTotal: "85899345920"
|
| 93 |
+
name: NVIDIA A100-SXM4-80GB
|
| 94 |
+
uuid: GPU-d9040f59-13d3-6c37-4ea0-c6c751424527
|
| 95 |
+
- architecture: Ampere
|
| 96 |
+
cudaCores: 6912
|
| 97 |
+
memoryTotal: "85899345920"
|
| 98 |
+
name: NVIDIA A100-SXM4-80GB
|
| 99 |
+
uuid: GPU-f834d0e8-a118-24c7-5461-30fce111bea6
|
| 100 |
+
- architecture: Ampere
|
| 101 |
+
cudaCores: 6912
|
| 102 |
+
memoryTotal: "85899345920"
|
| 103 |
+
name: NVIDIA A100-SXM4-80GB
|
| 104 |
+
uuid: GPU-6e687915-3247-735c-6641-544540804e79
|
| 105 |
+
host: nebula99
|
| 106 |
+
memory:
|
| 107 |
+
total: "1623177744384"
|
| 108 |
+
os: Linux-6.8.0-100-generic-x86_64-with-glibc2.39
|
| 109 |
+
program: /data/jliu/psi/scripts/train.py
|
| 110 |
+
python: CPython 3.10.20
|
| 111 |
+
root: /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426
|
| 112 |
+
startedAt: "2026-03-18T14:26:46.537089Z"
|
| 113 |
+
writerId: 0fepodeqeh00upnrzhqar4dfcf23o49l
|
| 114 |
+
m: []
|
| 115 |
+
python_version: 3.10.20
|
| 116 |
+
t:
|
| 117 |
+
"1":
|
| 118 |
+
- 1
|
| 119 |
+
- 11
|
| 120 |
+
- 41
|
| 121 |
+
- 49
|
| 122 |
+
- 71
|
| 123 |
+
"2":
|
| 124 |
+
- 1
|
| 125 |
+
- 11
|
| 126 |
+
- 41
|
| 127 |
+
- 49
|
| 128 |
+
- 51
|
| 129 |
+
- 71
|
| 130 |
+
- 83
|
| 131 |
+
"3":
|
| 132 |
+
- 2
|
| 133 |
+
- 13
|
| 134 |
+
- 61
|
| 135 |
+
"4": 3.10.20
|
| 136 |
+
"5": 0.25.1
|
| 137 |
+
"6": 4.57.0
|
| 138 |
+
"12": 0.25.1
|
| 139 |
+
"13": linux-x86_64
|
| 140 |
+
auto_tag_run:
|
| 141 |
+
value: false
|
| 142 |
+
data:
|
| 143 |
+
value:
|
| 144 |
+
root_dir: /data/jliu/data
|
| 145 |
+
train_repo_ids:
|
| 146 |
+
- G1WholebodyBendPick-v0
|
| 147 |
+
transform:
|
| 148 |
+
field:
|
| 149 |
+
action_max:
|
| 150 |
+
- 0.08620641380548477
|
| 151 |
+
- 0.13058121502399445
|
| 152 |
+
- 0.22948147356510162
|
| 153 |
+
- 0.020551620051264763
|
| 154 |
+
- 0.005824880674481392
|
| 155 |
+
- 0.010019193403422832
|
| 156 |
+
- 8.43817247186962e-07
|
| 157 |
+
- 0.39566752314567566
|
| 158 |
+
- 0
|
| 159 |
+
- 0
|
| 160 |
+
- 0.4860266447067261
|
| 161 |
+
- 1.0467392206192017
|
| 162 |
+
- 0.6470075845718384
|
| 163 |
+
- 0.8298009037971497
|
| 164 |
+
- 0.03516175225377083
|
| 165 |
+
- 0.11019192636013031
|
| 166 |
+
- 0.04779902100563049
|
| 167 |
+
- 0.12850724160671234
|
| 168 |
+
- 3.8688118365826085e-05
|
| 169 |
+
- 0.0012142359046265483
|
| 170 |
+
- 3.3343669201713055e-05
|
| 171 |
+
- 0.002679983852431178
|
| 172 |
+
- 0.00041063950629904866
|
| 173 |
+
- 0.1973484456539154
|
| 174 |
+
- 0.2633756697177887
|
| 175 |
+
- 0.34943076968193054
|
| 176 |
+
- 0.0012102096807211637
|
| 177 |
+
- 0.8342975974082947
|
| 178 |
+
- 0.31870752573013306
|
| 179 |
+
- 0.45533719658851624
|
| 180 |
+
- 0.15729404985904694
|
| 181 |
+
- 0.75
|
| 182 |
+
- 0
|
| 183 |
+
- 0
|
| 184 |
+
- 0
|
| 185 |
+
- 0
|
| 186 |
+
action_min:
|
| 187 |
+
- -0.13059291243553162
|
| 188 |
+
- -0.09108058363199234
|
| 189 |
+
- -0.0024844733998179436
|
| 190 |
+
- -0.20733775198459625
|
| 191 |
+
- -0.15850023925304413
|
| 192 |
+
- -0.17450474202632904
|
| 193 |
+
- -0.2997315526008606
|
| 194 |
+
- -0.015391111373901367
|
| 195 |
+
- -0.34571564197540283
|
| 196 |
+
- -0.4991437792778015
|
| 197 |
+
- 0
|
| 198 |
+
- 0
|
| 199 |
+
- 0
|
| 200 |
+
- 0
|
| 201 |
+
- -0.1015840545296669
|
| 202 |
+
- -0.06647031009197235
|
| 203 |
+
- -0.16578954458236694
|
| 204 |
+
- -0.14477218687534332
|
| 205 |
+
- -0.3665394186973572
|
| 206 |
+
- -0.28364259004592896
|
| 207 |
+
- -0.1775387078523636
|
| 208 |
+
- -0.48419490456581116
|
| 209 |
+
- -0.7551082968711853
|
| 210 |
+
- -0.2692946195602417
|
| 211 |
+
- -0.03164339065551758
|
| 212 |
+
- -3.876500704791397e-05
|
| 213 |
+
- -0.3909206688404083
|
| 214 |
+
- 0
|
| 215 |
+
- -0.04351663216948509
|
| 216 |
+
- -0.014203650876879692
|
| 217 |
+
- -0.049649015069007874
|
| 218 |
+
- 0.44999998807907104
|
| 219 |
+
- 0
|
| 220 |
+
- 0
|
| 221 |
+
- 0
|
| 222 |
+
- 0
|
| 223 |
+
action_norm_masks:
|
| 224 |
+
- true
|
| 225 |
+
- true
|
| 226 |
+
- true
|
| 227 |
+
- true
|
| 228 |
+
- true
|
| 229 |
+
- true
|
| 230 |
+
- false
|
| 231 |
+
action_norm_type: bounds
|
| 232 |
+
normalize_state: true
|
| 233 |
+
pad_action_dim: 36
|
| 234 |
+
pad_state_dim: 36
|
| 235 |
+
stat_action_key: action
|
| 236 |
+
stat_path: meta/stats_psi0.json
|
| 237 |
+
stat_state_key: states
|
| 238 |
+
state_max:
|
| 239 |
+
- 0.0860000029206276
|
| 240 |
+
- 0.2720000147819519
|
| 241 |
+
- 0.23100000619888306
|
| 242 |
+
- 0
|
| 243 |
+
- 6.510182259944486e-08
|
| 244 |
+
- 0
|
| 245 |
+
- 0
|
| 246 |
+
- 0.5550000071525574
|
| 247 |
+
- 0.02100004442036152
|
| 248 |
+
- 0
|
| 249 |
+
- 0.5429999828338623
|
| 250 |
+
- 1.13100004196167
|
| 251 |
+
- 0.5770000219345093
|
| 252 |
+
- 0.9580000042915344
|
| 253 |
+
- 0.07141251862049103
|
| 254 |
+
- 0.10899999737739563
|
| 255 |
+
- 0.04699999839067459
|
| 256 |
+
- 0.13600000739097595
|
| 257 |
+
- 0
|
| 258 |
+
- 0.003000000026077032
|
| 259 |
+
- 0
|
| 260 |
+
- 0.009999999776482582
|
| 261 |
+
- 0.0020000000949949026
|
| 262 |
+
- 0.2029999941587448
|
| 263 |
+
- 0.2759999930858612
|
| 264 |
+
- 0.3499999940395355
|
| 265 |
+
- 0.003000000026077032
|
| 266 |
+
- 0.8370000123977661
|
| 267 |
+
- 0
|
| 268 |
+
- 0
|
| 269 |
+
- 0
|
| 270 |
+
- 0.75
|
| 271 |
+
- 0
|
| 272 |
+
- 0
|
| 273 |
+
- 0
|
| 274 |
+
- 0
|
| 275 |
+
state_min:
|
| 276 |
+
- -0.13899999856948853
|
| 277 |
+
- -0.09099991619586945
|
| 278 |
+
- -5.989517215532203e-11
|
| 279 |
+
- -0.20900000631809235
|
| 280 |
+
- -0.1589999943971634
|
| 281 |
+
- -0.210999995470047
|
| 282 |
+
- -0.3009999990463257
|
| 283 |
+
- -0.01600000075995922
|
| 284 |
+
- -0.1860000044107437
|
| 285 |
+
- -0.6940000057220459
|
| 286 |
+
- 0
|
| 287 |
+
- 0
|
| 288 |
+
- 0
|
| 289 |
+
- 0
|
| 290 |
+
- -0.1019991859793663
|
| 291 |
+
- -0.06899992376565933
|
| 292 |
+
- -0.16899999976158142
|
| 293 |
+
- -0.14499999582767487
|
| 294 |
+
- -0.3709999918937683
|
| 295 |
+
- -0.28700000047683716
|
| 296 |
+
- -0.17800045013427734
|
| 297 |
+
- -0.4869999885559082
|
| 298 |
+
- -0.7599999904632568
|
| 299 |
+
- -0.27300000190734863
|
| 300 |
+
- -0.029999999329447746
|
| 301 |
+
- 0
|
| 302 |
+
- -0.39100033044815063
|
| 303 |
+
- -0.0010000000474974513
|
| 304 |
+
- 0
|
| 305 |
+
- -0.15000000596046448
|
| 306 |
+
- 0
|
| 307 |
+
- 0.44999998807907104
|
| 308 |
+
- 0
|
| 309 |
+
- 0
|
| 310 |
+
- 0
|
| 311 |
+
- 0
|
| 312 |
+
use_norm_mask: false
|
| 313 |
+
model:
|
| 314 |
+
center_crop:
|
| 315 |
+
size:
|
| 316 |
+
- 224
|
| 317 |
+
- 224
|
| 318 |
+
color_jitter:
|
| 319 |
+
brightness: 0.2
|
| 320 |
+
contrast:
|
| 321 |
+
- 0.8
|
| 322 |
+
- 1.2
|
| 323 |
+
hue: 0.05
|
| 324 |
+
saturation:
|
| 325 |
+
- 0.8
|
| 326 |
+
- 1.2
|
| 327 |
+
img_aug: true
|
| 328 |
+
normalize:
|
| 329 |
+
mean:
|
| 330 |
+
- 0.485
|
| 331 |
+
- 0.456
|
| 332 |
+
- 0.406
|
| 333 |
+
std:
|
| 334 |
+
- 0.229
|
| 335 |
+
- 0.224
|
| 336 |
+
- 0.225
|
| 337 |
+
resize:
|
| 338 |
+
size:
|
| 339 |
+
- 256
|
| 340 |
+
- 480
|
| 341 |
+
repack:
|
| 342 |
+
action_chunk_size: 30
|
| 343 |
+
dataset_name: simple
|
| 344 |
+
num_past_frames: 0
|
| 345 |
+
pad_action_dim: 36
|
| 346 |
+
pad_state_dim: 36
|
| 347 |
+
val_repo_ids:
|
| 348 |
+
- G1WholebodyBendPick-v0
|
| 349 |
+
debug:
|
| 350 |
+
value: false
|
| 351 |
+
environment_variables:
|
| 352 |
+
value:
|
| 353 |
+
CUDA_VISIBLE_DEVICES: 0,1,2,3
|
| 354 |
+
DATA_HOME: /data/data
|
| 355 |
+
HF_HOME: /data/cache
|
| 356 |
+
HF_LEROBOT_HOME: /data/data/lerobot
|
| 357 |
+
HF_TOKEN: hf_...TiKa
|
| 358 |
+
LOCAL_RANK: "0"
|
| 359 |
+
LOCAL_WORLD_SIZE: "4"
|
| 360 |
+
MASTER_ADDR: nebula99
|
| 361 |
+
MASTER_PORT: "37919"
|
| 362 |
+
OMP_NUM_THREADS: "32"
|
| 363 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
|
| 364 |
+
RANK: "0"
|
| 365 |
+
TORCH_HOME: /data/cache
|
| 366 |
+
UV_CACHE_DIR: /data/cache
|
| 367 |
+
WANDB_API_KEY: 90e...5c06
|
| 368 |
+
WE_HOME: Not Set
|
| 369 |
+
WORLD_SIZE: "4"
|
| 370 |
+
eval:
|
| 371 |
+
value: false
|
| 372 |
+
exp:
|
| 373 |
+
value: g1wholebodybendpick-v0
|
| 374 |
+
log:
|
| 375 |
+
value:
|
| 376 |
+
log_freq: 100
|
| 377 |
+
logging_dir: logs
|
| 378 |
+
report_to: wandb
|
| 379 |
+
model:
|
| 380 |
+
value:
|
| 381 |
+
action_dim: 36
|
| 382 |
+
chunk_size: 30
|
| 383 |
+
dim_feedforward: 3200
|
| 384 |
+
dim_model: 512
|
| 385 |
+
dropout: 0.1
|
| 386 |
+
feedforward_activation: relu
|
| 387 |
+
kl_weight: 10
|
| 388 |
+
latent_dim: 32
|
| 389 |
+
n_action_steps: 30
|
| 390 |
+
n_decoder_layers: 1
|
| 391 |
+
n_encoder_layers: 4
|
| 392 |
+
n_heads: 8
|
| 393 |
+
n_obs_steps: 1
|
| 394 |
+
n_vae_encoder_layers: 4
|
| 395 |
+
pre_norm: false
|
| 396 |
+
state_dim: 36
|
| 397 |
+
temporal_ensemble_coeff: null
|
| 398 |
+
use_vae: true
|
| 399 |
+
seed:
|
| 400 |
+
value: 2026
|
| 401 |
+
timestamp:
|
| 402 |
+
value: "2603181426"
|
| 403 |
+
train:
|
| 404 |
+
value:
|
| 405 |
+
checkpointing_steps: 5000
|
| 406 |
+
data_parallel: ddp
|
| 407 |
+
deepspeed_config: /data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json
|
| 408 |
+
enable_gradient_checkpointing: true
|
| 409 |
+
enable_mixed_precision_training: true
|
| 410 |
+
gradient_accumulation_steps: 1
|
| 411 |
+
hf_token: .hf_token
|
| 412 |
+
learning_rate: 0.0001
|
| 413 |
+
lora: false
|
| 414 |
+
lr_scheduler_kwargs:
|
| 415 |
+
betas:
|
| 416 |
+
- 0.95
|
| 417 |
+
- 0.999
|
| 418 |
+
eps: 1e-08
|
| 419 |
+
weight_decay: 1e-06
|
| 420 |
+
lr_scheduler_type: cosine
|
| 421 |
+
max_checkpoints_to_keep: null
|
| 422 |
+
max_grad_norm: 1
|
| 423 |
+
max_training_steps: 40000
|
| 424 |
+
mixed_precision: bf16
|
| 425 |
+
name: act-g1
|
| 426 |
+
num_train_epochs: null
|
| 427 |
+
num_workers: 8
|
| 428 |
+
optimizer_foreach: null
|
| 429 |
+
output_dir: .runs
|
| 430 |
+
overfit_single_batch: false
|
| 431 |
+
reduce_in_full_precision: true
|
| 432 |
+
resume_from_checkpoint: null
|
| 433 |
+
sharding_strategy: full-shard
|
| 434 |
+
skip_resumed_steps: false
|
| 435 |
+
train_batch_size: 32
|
| 436 |
+
val_batch_size: 16
|
| 437 |
+
val_num_batches: 20
|
| 438 |
+
validation_steps: 500
|
| 439 |
+
warmup_ratio: null
|
| 440 |
+
warmup_steps: 1000
|
| 441 |
+
wandb:
|
| 442 |
+
value:
|
| 443 |
+
entity: jliu530-soochow-university
|
| 444 |
+
group: null
|
| 445 |
+
id: null
|
| 446 |
+
name: null
|
| 447 |
+
project: psi
|
| 448 |
+
resume: allow
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/output.log
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2;36m[14:26:48 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Saved configuration to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m ]8;id=805427;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=514750;file:///data/jliu/psi/scripts/train.py#128\[2m128[0m]8;;\
|
| 2 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Training configurations: ]8;id=2657;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=644039;file:///data/jliu/psi/scripts/train.py#181\[2m181[0m]8;;\
|
| 3 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> training task: [32m'act-g1'[0m ]8;id=471497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=12046;file:///data/jliu/psi/scripts/train.py#182\[2m182[0m]8;;\
|
| 4 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> run name: g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m ]8;id=329497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=220436;file:///data/jliu/psi/scripts/train.py#183\[2m183[0m]8;;\
|
| 5 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> seed: [1;36m2026[0m ]8;id=844246;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=394625;file:///data/jliu/psi/scripts/train.py#184\[2m184[0m]8;;\
|
| 6 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=94020;file:///data/jliu/psi/scripts/train.py#185\[2m185[0m]8;;\
|
| 7 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> warmup steps: [1;36m1000[0m ]8;id=147732;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=919050;file:///data/jliu/psi/scripts/train.py#186\[2m186[0m]8;;\
|
| 8 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> validation steps: [1;36m500[0m ]8;id=25941;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=888976;file:///data/jliu/psi/scripts/train.py#187\[2m187[0m]8;;\
|
| 9 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> checkpoint steps: [1;36m5000[0m ]8;id=95089;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=418144;file:///data/jliu/psi/scripts/train.py#188\[2m188[0m]8;;\
|
| 10 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> max gradient norm: [1;36m1.0[0m ]8;id=122104;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=446109;file:///data/jliu/psi/scripts/train.py#189\[2m189[0m]8;;\
|
| 11 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> PyTorch version [1;36m2.7[0m.[1;36m0[0m available. ]8;id=514883;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py\[2mconfig.py[0m]8;;\[2m:[0m]8;id=417488;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py#54\[2m54[0m]8;;\
|
| 12 |
+
Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 206108.30it/s]
|
| 13 |
+
Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 191084.46it/s]
|
| 14 |
+
[2;36m[14:26:50 03/18][0m[2;36m [0m[34mINFO [0m | >> |=> Num training samples: ]8;id=591396;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=971225;file:///data/jliu/psi/scripts/train.py#192\[2m192[0m]8;;\
|
| 15 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Training dataset size: [1;36m15[0m,[1;36m509[0m ]8;id=603472;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=235431;file:///data/jliu/psi/scripts/train.py#193\[2m193[0m]8;;\
|
| 16 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Val dataset size: [1;36m15[0m,[1;36m509[0m ]8;id=548632;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=929903;file:///data/jliu/psi/scripts/train.py#195\[2m195[0m]8;;\
|
| 17 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Initialize optimizers and schedulers[33m...[0m ]8;id=492873;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=149330;file:///data/jliu/psi/scripts/train.py#198\[2m198[0m]8;;\
|
| 18 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m ***** Running training ***** ]8;id=127940;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=657111;file:///data/jliu/psi/scripts/train.py#202\[2m202[0m]8;;\
|
| 19 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num training examples = [1;36m15509[0m ]8;id=974614;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=572136;file:///data/jliu/psi/scripts/train.py#203\[2m203[0m]8;;\
|
| 20 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Max training Epochs = [1;36m328[0m ]8;id=574775;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=838653;file:///data/jliu/psi/scripts/train.py#204\[2m204[0m]8;;\
|
| 21 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Total optimization steps = [1;36m40000[0m ]8;id=927413;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=421473;file:///data/jliu/psi/scripts/train.py#205\[2m205[0m]8;;\
|
| 22 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num steps Per Epoch = [1;36m122[0m ]8;id=9727;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=34717;file:///data/jliu/psi/scripts/train.py#206\[2m206[0m]8;;\
|
| 23 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Effective training epochs = [1;36m1311.48[0m ]8;id=842580;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=548288;file:///data/jliu/psi/scripts/train.py#207\[2m207[0m]8;;\
|
| 24 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Global train batch size [1m([0mw. parallel, distributed & accumulation[1m)[0m = [1;36m128[0m ]8;id=272745;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=726293;file:///data/jliu/psi/scripts/train.py#208\[2m208[0m]8;;\
|
| 25 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Device train batch size = [1;36m32[0m ]8;id=521307;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=847206;file:///data/jliu/psi/scripts/train.py#209\[2m209[0m]8;;\
|
| 26 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Gradient Accumulation steps = [1;36m1[0m ]8;id=871789;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=304159;file:///data/jliu/psi/scripts/train.py#210\[2m210[0m]8;;\
|
| 27 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num processes [1m([0mGPUs[1m)[0m = [1;36m4[0m ]8;id=459749;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=722818;file:///data/jliu/psi/scripts/train.py#211\[2m211[0m]8;;\
|
| 28 |
+
[2;36m[14:26:51 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Accelerator runs in: .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m ]8;id=966756;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=191940;file:///data/jliu/psi/scripts/train.py#218\[2m218[0m]8;;\
|
| 29 |
+
Traing steps: 25%|██████████████████████▍ | 9999/40000 [1:25:29<2:57:03, 2.82it/s, loss=0.0482, lr=8.7e-05]
|
| 30 |
+
[2026-03-18 14:26:59,782] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 31 |
+
[2;36m[14:26:59 03/18][0m[2;36m [0m[34mINFO [0m | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=680052;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=279409;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 32 |
+
[2;36m [0m [35m/tmp/tmpb55feb63/[0m[95mtest.c[0m -o [35m/tmp/tmpb55feb63/[0m[95mtest.o[0m [2m [0m
|
| 33 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread [35m/tmp/tmpb55feb63/[0m[95mtest.o[0m -laio -o [35m/tmp/tmpb55feb63/[0m[95ma.out[0m ]8;id=204038;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=783864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 34 |
+
[2;36m[14:27:00 03/18][0m[2;36m [0m[34mINFO [0m | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=960999;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=14714;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 35 |
+
[2;36m [0m [35m/tmp/tmp54jhxf13/[0m[95mtest.c[0m -o [35m/tmp/tmp54jhxf13/[0m[95mtest.o[0m [2m [0m
|
| 36 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread [35m/tmp/tmp54jhxf13/[0m[95mtest.o[0m -L/usr/local/cuda -L/usr/local/cuda/lib64 -lcufile -o [35m/tmp/tmp54jhxf13/[0m[95ma.out[0m ]8;id=573141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=977459;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 37 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=208448;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=259876;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 38 |
+
[2;36m [0m [35m/tmp/tmpksfo7p26/[0m[95mtest.c[0m -o [35m/tmp/tmpksfo7p26/[0m[95mtest.o[0m [2m [0m
|
| 39 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread [35m/tmp/tmpksfo7p26/[0m[95mtest.o[0m -laio -o [35m/tmp/tmpksfo7p26/[0m[95ma.out[0m ]8;id=572992;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=281580;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 40 |
+
[2026-03-18 14:27:01,308] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
|
| 41 |
+
|
| 42 |
+
[2;36m[15:09:33 03/18][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=328668;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=696864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 43 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000 [2m [0m
|
| 44 |
+
[2;36m[15:09:34 03/18][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=515887;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=723262;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 45 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/model.safeten [2m [0m
|
| 46 |
+
[2;36m [0m sors [2m [0m
|
| 47 |
+
[2;36m[15:09:35 03/18][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=495959;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=616494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 48 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/optimizer.bin [2m [0m
|
| 49 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=304800;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=88534;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 50 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/scheduler.bin [2m [0m
|
| 51 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=821494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=820417;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 52 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/sampler.bin [2m [0m
|
| 53 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=432054;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=409371;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 54 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/sampler_1.bin [2m [0m
|
| 55 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=394271;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=14818;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 56 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/random_states [2m [0m
|
| 57 |
+
[2;36m [0m _0.pkl [2m [0m
|
| 58 |
+
Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000
|
| 59 |
+
[2;36m[15:52:21 03/18][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=53870;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=874302;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 60 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000 [2m [0m
|
| 61 |
+
[2;36m[15:52:24 03/18][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=806914;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=990978;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 62 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000/model.safete [2m [0m
|
| 63 |
+
[2;36m [0m nsors [2m [0m
|
| 64 |
+
[2;36m[15:52:25 03/18][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=343681;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=715961;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 65 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000/optimizer.bi [2m [0m
|
| 66 |
+
[2;36m [0m n [2m [0m
|
| 67 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=127268;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=521739;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 68 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000/scheduler.bi [2m [0m
|
| 69 |
+
[2;36m [0m n [2m [0m
|
| 70 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=844039;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=179027;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 71 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000/sampler.bin [2m [0m
|
| 72 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=454670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=653622;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 73 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000/sampler_1.bi [2m [0m
|
| 74 |
+
[2;36m [0m n [2m [0m
|
| 75 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=711457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=745414;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 76 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000/random_state [2m [0m
|
| 77 |
+
[2;36m [0m s_0.pkl [2m [0m
|
| 78 |
+
Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000
|
| 79 |
+
[2;36m[16:33:16 03/18][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=204990;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=678670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 80 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000 [2m [0m
|
| 81 |
+
[2;36m[16:33:17 03/18][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=571229;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=976178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 82 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000/model.safete [2m [0m
|
| 83 |
+
[2;36m [0m nsors [2m [0m
|
| 84 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=791370;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=671982;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 85 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000/optimizer.bi [2m [0m
|
| 86 |
+
[2;36m [0m n [2m [0m
|
| 87 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=920457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=21464;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 88 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000/scheduler.bi [2m [0m
|
| 89 |
+
[2;36m [0m n [2m [0m
|
| 90 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=823560;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=925542;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 91 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000/sampler.bin [2m [0m
|
| 92 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=693342;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=885391;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 93 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000/sampler_1.bi [2m [0m
|
| 94 |
+
[2;36m [0m n [2m [0m
|
| 95 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=558902;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=434201;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 96 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000/random_state [2m [0m
|
| 97 |
+
[2;36m [0m s_0.pkl [2m [0m
|
| 98 |
+
Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000
|
| 99 |
+
[2;36m[17:14:14 03/18][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=196292;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=360779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 100 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000 [2m [0m
|
| 101 |
+
[2;36m[17:14:16 03/18][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=818149;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=854583;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 102 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000/model.safete [2m [0m
|
| 103 |
+
[2;36m [0m nsors [2m [0m
|
| 104 |
+
[2;36m[17:14:17 03/18][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=341614;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=544562;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 105 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000/optimizer.bi [2m [0m
|
| 106 |
+
[2;36m [0m n [2m [0m
|
| 107 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=835298;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=550186;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 108 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000/scheduler.bi [2m [0m
|
| 109 |
+
[2;36m [0m n [2m [0m
|
| 110 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=889056;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=224742;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 111 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000/sampler.bin [2m [0m
|
| 112 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=921300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=476172;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 113 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000/sampler_1.bi [2m [0m
|
| 114 |
+
[2;36m [0m n [2m [0m
|
| 115 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=821364;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=874045;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 116 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000/random_state [2m [0m
|
| 117 |
+
[2;36m [0m s_0.pkl [2m [0m
|
| 118 |
+
Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000
|
| 119 |
+
[2;36m[17:54:01 03/18][0m[2;36m [0m[33mWARNING [0m | >> [1m[[0m*[1m][0m Checkpoint [1;36m25000[0m already exists, skipping save. ]8;id=369046;file:///data/jliu/psi/src/psi/trainers/act_g1.py\[2mact_g1.py[0m]8;;\[2m:[0m]8;id=887661;file:///data/jliu/psi/src/psi/trainers/act_g1.py#177\[2m177[0m]8;;\
|
| 120 |
+
Saved state to None
|
| 121 |
+
[2;36m[18:33:46 03/18][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=600133;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=830265;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 122 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000 [2m [0m
|
| 123 |
+
[2;36m[18:33:47 03/18][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=750793;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=294096;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 124 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000/model.safete [2m [0m
|
| 125 |
+
[2;36m [0m nsors [2m [0m
|
| 126 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=224851;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=597515;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 127 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000/optimizer.bi [2m [0m
|
| 128 |
+
[2;36m [0m n [2m [0m
|
| 129 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=582729;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=139060;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 130 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000/scheduler.bi [2m [0m
|
| 131 |
+
[2;36m [0m n [2m [0m
|
| 132 |
+
[2;36m[18:33:48 03/18][0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=457234;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=179594;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 133 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000/sampler.bin [2m [0m
|
| 134 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=373558;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=757447;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 135 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000/sampler_1.bi [2m [0m
|
| 136 |
+
[2;36m [0m n [2m [0m
|
| 137 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=111484;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=531236;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 138 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000/random_state [2m [0m
|
| 139 |
+
[2;36m [0m s_0.pkl [2m [0m
|
| 140 |
+
Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000
|
| 141 |
+
[2;36m[19:13:36 03/18][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=67019;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=305173;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 142 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000 [2m [0m
|
| 143 |
+
[2;36m[19:13:37 03/18][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=268565;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=65933;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 144 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000/model.safete [2m [0m
|
| 145 |
+
[2;36m [0m nsors [2m [0m
|
| 146 |
+
[2;36m[19:13:38 03/18][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=948866;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=814512;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 147 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000/optimizer.bi [2m [0m
|
| 148 |
+
[2;36m [0m n [2m [0m
|
| 149 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=101135;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=246651;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 150 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000/scheduler.bi [2m [0m
|
| 151 |
+
[2;36m [0m n [2m [0m
|
| 152 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=719141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=588438;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 153 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000/sampler.bin [2m [0m
|
| 154 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=330274;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=817634;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 155 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000/sampler_1.bi [2m [0m
|
| 156 |
+
[2;36m [0m n [2m [0m
|
| 157 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=748910;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=645026;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 158 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000/random_state [2m [0m
|
| 159 |
+
[2;36m [0m s_0.pkl [2m [0m
|
| 160 |
+
Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000
|
| 161 |
+
[2;36m[19:53:30 03/18][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=637309;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=226110;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 162 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000 [2m [0m
|
| 163 |
+
[2;36m[19:53:32 03/18][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=921072;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=125242;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 164 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000/model.safete [2m [0m
|
| 165 |
+
[2;36m [0m nsors [2m [0m
|
| 166 |
+
[2;36m[19:53:33 03/18][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=714503;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=968603;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 167 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000/optimizer.bi [2m [0m
|
| 168 |
+
[2;36m [0m n [2m [0m
|
| 169 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=856493;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=373626;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 170 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000/scheduler.bi [2m [0m
|
| 171 |
+
[2;36m [0m n [2m [0m
|
| 172 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=921087;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=24812;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 173 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000/sampler.bin [2m [0m
|
| 174 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=762447;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=806512;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 175 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000/sampler_1.bi [2m [0m
|
| 176 |
+
[2;36m [0m n [2m [0m
|
| 177 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=805194;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=436875;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 178 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000/random_state [2m [0m
|
| 179 |
+
[2;36m [0m s_0.pkl [2m [0m
|
| 180 |
+
Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000
|
| 181 |
+
Training has reached maximum steps.
|
| 182 |
+
[2;36m[19:53:46 03/18][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=568492;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=359120;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 183 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999 [2m [0m
|
| 184 |
+
[2;36m[19:53:47 03/18][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=118067;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=169999;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 185 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999/model.safete [2m [0m
|
| 186 |
+
[2;36m [0m nsors [2m [0m
|
| 187 |
+
[2;36m[19:53:48 03/18][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=131787;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=547968;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 188 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999/optimizer.bi [2m [0m
|
| 189 |
+
[2;36m [0m n [2m [0m
|
| 190 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=316922;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=30192;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 191 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999/scheduler.bi [2m [0m
|
| 192 |
+
[2;36m [0m n [2m [0m
|
| 193 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=435471;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=402764;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 194 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999/sampler.bin [2m [0m
|
| 195 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=694779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=230230;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 196 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999/sampler_1.bi [2m [0m
|
| 197 |
+
[2;36m [0m n [2m [0m
|
| 198 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=29797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=693698;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 199 |
+
[2;36m [0m .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999/random_state [2m [0m
|
| 200 |
+
[2;36m [0m s_0.pkl [2m [0m
|
| 201 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Finalized ACT Trainer. Epoch losses: [1m[[0m[1;36m0.0[0m, [1;36m4.91099214553833[0m, [1;36m2.8780977725982666[0m, [1;36m2.0492076873779297[0m, ]8;id=416136;file:///data/jliu/psi/src/psi/trainers/act_g1.py\[2mact_g1.py[0m]8;;\[2m:[0m]8;id=932337;file:///data/jliu/psi/src/psi/trainers/act_g1.py#331\[2m331[0m]8;;\
|
| 202 |
+
[2;36m [0m [1;36m1.8726189136505127[0m, [1;36m1.5002448558807373[0m[1m][0m [2m [0m
|
| 203 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Happy Ending! ]8;id=17401;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=290278;file:///data/jliu/psi/scripts/train.py#310\[2m310[0m]8;;\
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/requirements.txt
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
tifffile==2025.5.10
|
| 2 |
+
mpmath==1.3.0
|
| 3 |
+
pynput==1.8.1
|
| 4 |
+
exceptiongroup==1.3.0
|
| 5 |
+
sympy==1.14.0
|
| 6 |
+
aiohttp-cors==0.8.1
|
| 7 |
+
soupsieve==2.8
|
| 8 |
+
Farama-Notifications==0.0.4
|
| 9 |
+
numpydantic==1.6.7
|
| 10 |
+
uvicorn==0.38.0
|
| 11 |
+
waterbear==2.6.8
|
| 12 |
+
cmake==4.2.3
|
| 13 |
+
xxhash==3.5.0
|
| 14 |
+
nvidia-cusparse-cu12==12.5.4.2
|
| 15 |
+
annotated-doc==0.0.4
|
| 16 |
+
jsonlines==4.0.0
|
| 17 |
+
nvidia-cuda-runtime-cu12==12.6.77
|
| 18 |
+
pydantic==2.10.6
|
| 19 |
+
multidict==6.6.4
|
| 20 |
+
shtab==1.7.2
|
| 21 |
+
nvidia-cuda-nvrtc-cu12==12.6.77
|
| 22 |
+
pycollada==0.9.2
|
| 23 |
+
beautifulsoup4==4.14.2
|
| 24 |
+
msgspec==0.19.0
|
| 25 |
+
huggingface-hub==0.35.3
|
| 26 |
+
pydantic_core==2.27.2
|
| 27 |
+
pytz==2025.2
|
| 28 |
+
gymnasium==1.2.3
|
| 29 |
+
attrs==25.3.0
|
| 30 |
+
requests==2.32.5
|
| 31 |
+
hf_transfer==0.1.9
|
| 32 |
+
einx==0.3.0
|
| 33 |
+
scipy==1.15.3
|
| 34 |
+
transforms3d==0.4.2
|
| 35 |
+
aiohappyeyeballs==2.6.1
|
| 36 |
+
six==1.17.0
|
| 37 |
+
expandvars==1.1.2
|
| 38 |
+
prompt_toolkit==3.0.52
|
| 39 |
+
wcwidth==0.6.0
|
| 40 |
+
ruamel.yaml.clib==0.2.14
|
| 41 |
+
multiprocess==0.70.16
|
| 42 |
+
vhacdx==0.0.9
|
| 43 |
+
tokenizers==0.22.2
|
| 44 |
+
aiosignal==1.4.0
|
| 45 |
+
itsdangerous==2.2.0
|
| 46 |
+
torchvision==0.22.0
|
| 47 |
+
nvidia-cublas-cu12==12.6.4.1
|
| 48 |
+
httpx==0.28.1
|
| 49 |
+
platformdirs==4.4.0
|
| 50 |
+
sentry-sdk==2.39.0
|
| 51 |
+
plotly==6.2.0
|
| 52 |
+
nvidia-cudnn-cu12==9.5.1.17
|
| 53 |
+
jsonschema==4.25.1
|
| 54 |
+
fastapi==0.119.1
|
| 55 |
+
fsspec==2025.3.0
|
| 56 |
+
MarkupSafe==3.0.3
|
| 57 |
+
pydantic-yaml==1.6.0
|
| 58 |
+
h11==0.16.0
|
| 59 |
+
typer==0.24.1
|
| 60 |
+
mypy_extensions==1.1.0
|
| 61 |
+
h5py==3.14.0
|
| 62 |
+
python-xlib==0.33
|
| 63 |
+
lazy_loader==0.4
|
| 64 |
+
einops==0.8.1
|
| 65 |
+
albumentations==1.4.18
|
| 66 |
+
params_proto==2.13.2
|
| 67 |
+
psutil==7.1.0
|
| 68 |
+
starlette==0.48.0
|
| 69 |
+
anyio==4.11.0
|
| 70 |
+
gdown==5.2.0
|
| 71 |
+
charset-normalizer==3.4.3
|
| 72 |
+
tyro==0.9.32
|
| 73 |
+
filelock==3.19.1
|
| 74 |
+
websockets==15.0.1
|
| 75 |
+
nvidia-cuda-cupti-cu12==12.6.80
|
| 76 |
+
python-dotenv==1.2.1
|
| 77 |
+
orderly-set==5.5.0
|
| 78 |
+
inquirerpy==0.3.4
|
| 79 |
+
urllib3==2.5.0
|
| 80 |
+
diffusers==0.37.0
|
| 81 |
+
PyYAML==6.0.3
|
| 82 |
+
antlr4-python3-runtime==4.9.3
|
| 83 |
+
mdurl==0.1.2
|
| 84 |
+
omegaconf==2.3.0
|
| 85 |
+
rerun-sdk==0.22.1
|
| 86 |
+
draccus==0.10.0
|
| 87 |
+
sentencepiece==0.2.1
|
| 88 |
+
referencing==0.37.0
|
| 89 |
+
docstring_parser==0.17.0
|
| 90 |
+
protobuf==6.33.5
|
| 91 |
+
wandb==0.25.1
|
| 92 |
+
numpy==1.26.4
|
| 93 |
+
GitPython==3.1.45
|
| 94 |
+
opencv-python-headless==4.11.0.86
|
| 95 |
+
yourdfpy==0.0.58
|
| 96 |
+
async-timeout==4.0.3
|
| 97 |
+
shapely==2.1.2
|
| 98 |
+
frozenlist==1.7.0
|
| 99 |
+
simplejpeg==1.9.0
|
| 100 |
+
Pygments==2.19.2
|
| 101 |
+
py-cpuinfo==9.0.0
|
| 102 |
+
rtree==1.4.1
|
| 103 |
+
gitdb==4.0.12
|
| 104 |
+
cloudpickle==3.1.1
|
| 105 |
+
deepspeed==0.17.1
|
| 106 |
+
nvidia-nvjitlink-cu12==12.6.85
|
| 107 |
+
colorlog==6.10.1
|
| 108 |
+
qwen-vl-utils==0.0.14
|
| 109 |
+
Werkzeug==3.1.6
|
| 110 |
+
zipp==3.23.0
|
| 111 |
+
setuptools==80.9.0
|
| 112 |
+
albucore==0.0.17
|
| 113 |
+
transformers==4.57.0
|
| 114 |
+
imageio==2.34.2
|
| 115 |
+
frozendict==2.4.6
|
| 116 |
+
hjson==3.1.0
|
| 117 |
+
jsonschema-specifications==2025.9.1
|
| 118 |
+
rpds-py==0.28.0
|
| 119 |
+
blinker==1.9.0
|
| 120 |
+
accelerate==1.7.0
|
| 121 |
+
argparse==1.4.0
|
| 122 |
+
msgpack==1.1.1
|
| 123 |
+
smmap==5.0.2
|
| 124 |
+
nvidia-nvtx-cu12==12.6.77
|
| 125 |
+
packaging==25.0
|
| 126 |
+
embreex==2.17.7.post7
|
| 127 |
+
vuer==0.0.68
|
| 128 |
+
propcache==0.3.2
|
| 129 |
+
nvidia-cusolver-cu12==11.7.1.2
|
| 130 |
+
triton==3.3.0
|
| 131 |
+
pfzy==0.3.4
|
| 132 |
+
torchcodec==0.4.0
|
| 133 |
+
dm-tree==0.1.8
|
| 134 |
+
ninja==1.13.0
|
| 135 |
+
nvidia-cufile-cu12==1.11.1.6
|
| 136 |
+
datasets==3.6.0
|
| 137 |
+
nvidia-curand-cu12==10.3.7.77
|
| 138 |
+
flash_attn==2.7.4.post1
|
| 139 |
+
av==16.0.1
|
| 140 |
+
sniffio==1.3.1
|
| 141 |
+
Jinja2==3.1.6
|
| 142 |
+
narwhals==2.6.0
|
| 143 |
+
yarl==1.20.1
|
| 144 |
+
typeguard==4.4.4
|
| 145 |
+
termcolor==3.1.0
|
| 146 |
+
importlib_metadata==8.7.1
|
| 147 |
+
eval_type_backport==0.2.2
|
| 148 |
+
certifi==2025.8.3
|
| 149 |
+
tzdata==2025.2
|
| 150 |
+
typing_extensions==4.15.0
|
| 151 |
+
viser==1.0.15
|
| 152 |
+
typing-inspect==0.9.0
|
| 153 |
+
annotated-types==0.7.0
|
| 154 |
+
svg.path==7.0
|
| 155 |
+
mergedeep==1.3.4
|
| 156 |
+
psi==0.0.0
|
| 157 |
+
deepdiff==8.6.1
|
| 158 |
+
toml==0.10.2
|
| 159 |
+
click==8.3.0
|
| 160 |
+
dotenv==0.9.9
|
| 161 |
+
trimesh==4.8.3
|
| 162 |
+
nvidia-nccl-cu12==2.26.2
|
| 163 |
+
pyyaml-include==1.4.1
|
| 164 |
+
scikit-image==0.25.2
|
| 165 |
+
PySocks==1.7.1
|
| 166 |
+
aiohttp==3.10.5
|
| 167 |
+
opencv-python==4.11.0.86
|
| 168 |
+
nodeenv==1.9.1
|
| 169 |
+
lerobot==0.3.3
|
| 170 |
+
pillow==11.3.0
|
| 171 |
+
idna==3.10
|
| 172 |
+
Flask==3.1.3
|
| 173 |
+
killport==1.2.0
|
| 174 |
+
hf-xet==1.1.10
|
| 175 |
+
imageio-ffmpeg==0.6.0
|
| 176 |
+
httpcore==1.0.9
|
| 177 |
+
nvidia-cusparselt-cu12==0.6.3
|
| 178 |
+
networkx==3.4.2
|
| 179 |
+
ruamel.yaml==0.18.15
|
| 180 |
+
manifold3d==3.2.1
|
| 181 |
+
shellingham==1.5.4
|
| 182 |
+
nvidia-cufft-cu12==11.3.0.4
|
| 183 |
+
mapbox_earcut==1.0.3
|
| 184 |
+
peft==0.17.1
|
| 185 |
+
pyarrow==21.0.0
|
| 186 |
+
pyserial==3.5
|
| 187 |
+
torch==2.7.0
|
| 188 |
+
dill==0.3.8
|
| 189 |
+
markdown-it-py==4.0.0
|
| 190 |
+
tqdm==4.67.1
|
| 191 |
+
argcomplete==3.6.2
|
| 192 |
+
pandas==2.3.3
|
| 193 |
+
lxml==6.0.2
|
| 194 |
+
safetensors==0.6.2
|
| 195 |
+
python-dateutil==2.9.0.post0
|
| 196 |
+
regex==2025.9.18
|
| 197 |
+
evdev==1.9.3
|
| 198 |
+
rich==14.1.0
|
| 199 |
+
psi==0.0.0
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-03-18T14:26:46.537089Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"simple_act_config",
|
| 7 |
+
"--seed=2026",
|
| 8 |
+
"--exp=g1wholebodybendpick-v0",
|
| 9 |
+
"--train.name=act-g1",
|
| 10 |
+
"--log.report-to=wandb",
|
| 11 |
+
"--train.data_parallel=ddp",
|
| 12 |
+
"--train.mixed_precision=bf16",
|
| 13 |
+
"--train.train-batch-size=32",
|
| 14 |
+
"--train.warmup-steps=1000",
|
| 15 |
+
"--train.warmup-ratio=None",
|
| 16 |
+
"--train.checkpointing-steps=5000",
|
| 17 |
+
"--train.validation_steps=500",
|
| 18 |
+
"--train.val_num_batches=20",
|
| 19 |
+
"--train.gradient_accumulation_steps=1",
|
| 20 |
+
"--train.max-training-steps=40000",
|
| 21 |
+
"--train.learning-rate=1e-4",
|
| 22 |
+
"--train.max-grad-norm=1.0",
|
| 23 |
+
"--train.lr_scheduler_kwargs.weight_decay=1e-6",
|
| 24 |
+
"--train.lr_scheduler_kwargs.betas",
|
| 25 |
+
"0.95",
|
| 26 |
+
"0.999",
|
| 27 |
+
"--train.lr_scheduler_type=cosine",
|
| 28 |
+
"--data.root_dir=/data/jliu/data",
|
| 29 |
+
"--data.train-repo-ids=G1WholebodyBendPick-v0",
|
| 30 |
+
"--data.transform.repack.action_chunk_size=30",
|
| 31 |
+
"--data.transform.repack.pad-action-dim=36",
|
| 32 |
+
"--data.transform.repack.pad-state-dim=36",
|
| 33 |
+
"--data.transform.field.stat-path=meta/stats_psi0.json",
|
| 34 |
+
"--data.transform.field.stat-action-key=action",
|
| 35 |
+
"--data.transform.field.stat-state-key=states",
|
| 36 |
+
"--data.transform.field.normalize-state",
|
| 37 |
+
"--data.transform.field.action-norm-type=bounds",
|
| 38 |
+
"--data.transform.field.pad-action-dim=36",
|
| 39 |
+
"--data.transform.field.pad-state-dim=36",
|
| 40 |
+
"--data.transform.model.img-aug",
|
| 41 |
+
"--model.chunk-size=30",
|
| 42 |
+
"--model.n-action-steps=30",
|
| 43 |
+
"--model.action-dim=36",
|
| 44 |
+
"--model.state-dim=36",
|
| 45 |
+
"--model.use-vae",
|
| 46 |
+
"--model.kl-weight=10.0"
|
| 47 |
+
],
|
| 48 |
+
"program": "/data/jliu/psi/scripts/train.py",
|
| 49 |
+
"codePath": "scripts/train.py",
|
| 50 |
+
"codePathLocal": "scripts/train.py",
|
| 51 |
+
"git": {
|
| 52 |
+
"remote": "https://github.com/songlin/psi.git",
|
| 53 |
+
"commit": "04614628ecb677f5f278e2d31c1103aed8127f26"
|
| 54 |
+
},
|
| 55 |
+
"email": "jliu530@163.com",
|
| 56 |
+
"root": "/data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426",
|
| 57 |
+
"host": "nebula99",
|
| 58 |
+
"executable": "/data/jliu/psi/.venv-psi/bin/python3",
|
| 59 |
+
"cpu_count": 128,
|
| 60 |
+
"cpu_count_logical": 128,
|
| 61 |
+
"gpu": "NVIDIA A100-SXM4-80GB",
|
| 62 |
+
"gpu_count": 8,
|
| 63 |
+
"disk": {
|
| 64 |
+
"/": {
|
| 65 |
+
"total": "105089261568",
|
| 66 |
+
"used": "99538649088"
|
| 67 |
+
}
|
| 68 |
+
},
|
| 69 |
+
"memory": {
|
| 70 |
+
"total": "1623177744384"
|
| 71 |
+
},
|
| 72 |
+
"gpu_nvidia": [
|
| 73 |
+
{
|
| 74 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 75 |
+
"memoryTotal": "85899345920",
|
| 76 |
+
"cudaCores": 6912,
|
| 77 |
+
"architecture": "Ampere",
|
| 78 |
+
"uuid": "GPU-37370f00-90f5-4e03-f763-7a35649e6783"
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 82 |
+
"memoryTotal": "85899345920",
|
| 83 |
+
"cudaCores": 6912,
|
| 84 |
+
"architecture": "Ampere",
|
| 85 |
+
"uuid": "GPU-d4b40383-188b-cc16-9180-20c1a71a777f"
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 89 |
+
"memoryTotal": "85899345920",
|
| 90 |
+
"cudaCores": 6912,
|
| 91 |
+
"architecture": "Ampere",
|
| 92 |
+
"uuid": "GPU-0f3d827c-66cf-04da-b182-4ed0414a2549"
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 96 |
+
"memoryTotal": "85899345920",
|
| 97 |
+
"cudaCores": 6912,
|
| 98 |
+
"architecture": "Ampere",
|
| 99 |
+
"uuid": "GPU-164a49bb-43d3-2250-59e2-1b40eee0757f"
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 103 |
+
"memoryTotal": "85899345920",
|
| 104 |
+
"cudaCores": 6912,
|
| 105 |
+
"architecture": "Ampere",
|
| 106 |
+
"uuid": "GPU-c0471f40-ae19-5371-a1af-da406ca30f83"
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 110 |
+
"memoryTotal": "85899345920",
|
| 111 |
+
"cudaCores": 6912,
|
| 112 |
+
"architecture": "Ampere",
|
| 113 |
+
"uuid": "GPU-d9040f59-13d3-6c37-4ea0-c6c751424527"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 117 |
+
"memoryTotal": "85899345920",
|
| 118 |
+
"cudaCores": 6912,
|
| 119 |
+
"architecture": "Ampere",
|
| 120 |
+
"uuid": "GPU-f834d0e8-a118-24c7-5461-30fce111bea6"
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 124 |
+
"memoryTotal": "85899345920",
|
| 125 |
+
"cudaCores": 6912,
|
| 126 |
+
"architecture": "Ampere",
|
| 127 |
+
"uuid": "GPU-6e687915-3247-735c-6641-544540804e79"
|
| 128 |
+
}
|
| 129 |
+
],
|
| 130 |
+
"cudaVersion": "12.9",
|
| 131 |
+
"writerId": "0fepodeqeh00upnrzhqar4dfcf23o49l"
|
| 132 |
+
}
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"val/denorm_err_l1_torso_vyaw":0,"val/denorm_err_l1_rpy":0.0013011818518862128,"val/kld_loss":9.509166702628135e-05,"val/denorm_err_l1_hand_joints":0.0025742806028574705,"_wandb":{"runtime":19620},"train/kld_loss":0.0014440594241023064,"val/bc_loss":0.004661495331674814,"val/denorm_err_l1_torso_vx":0,"val/denorm_err_l1_torso_target_yaw":0,"train/l1_loss":0.0042681763879954815,"train/loss":0.018708771094679832,"_timestamp":1.7738636260787222e+09,"val/denorm_err_l1_height":0.00026183543377555907,"train/epoch":330,"val/denorm_err_l1_arm_joints":0.0026575601659715176,"_step":40000,"val/denorm_err_l1_torso_vy":0,"_runtime":19620.194410914}
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-03-18 14:26:46,542 INFO MainThread:280442 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-03-18 14:26:46,542 INFO MainThread:280442 [wandb_setup.py:_flush():81] Configure stats pid to 280442
|
| 3 |
+
2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug.log
|
| 5 |
+
2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug-internal.log
|
| 6 |
+
2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-03-18 14:26:46,802 INFO MainThread:280442 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-03-18 14:26:46,814 INFO MainThread:280442 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-03-18 14:26:46,815 INFO MainThread:280442 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-03-18 14:26:46,822 INFO MainThread:280442 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-03-18 14:26:48,282 INFO MainThread:280442 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-03-18 14:26:48,444 INFO MainThread:280442 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-03-18 14:26:48,451 INFO MainThread:280442 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-03-18 14:26:48,454 INFO MainThread:280442 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodybendpick-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2603181426', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.13059291243553162, -0.09108058363199234, -0.0024844733998179436, -0.20733775198459625, -0.15850023925304413, -0.17450474202632904, -0.2997315526008606, -0.015391111373901367, -0.34571564197540283, -0.4991437792778015, 0.0, 0.0, 0.0, 0.0, -0.1015840545296669, -0.06647031009197235, -0.16578954458236694, -0.14477218687534332, -0.3665394186973572, -0.28364259004592896, -0.1775387078523636, -0.48419490456581116, -0.7551082968711853, -0.2692946195602417, -0.03164339065551758, -3.876500704791397e-05, -0.3909206688404083, 0.0, -0.04351663216948509, -0.014203650876879692, -0.049649015069007874, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.08620641380548477, 0.13058121502399445, 0.22948147356510162, 0.020551620051264763, 0.005824880674481392, 0.010019193403422832, 8.43817247186962e-07, 0.39566752314567566, 0.0, 0.0, 0.4860266447067261, 1.0467392206192017, 0.6470075845718384, 0.8298009037971497, 0.03516175225377083, 0.11019192636013031, 0.04779902100563049, 0.12850724160671234, 3.8688118365826085e-05, 0.0012142359046265483, 3.3343669201713055e-05, 0.002679983852431178, 0.00041063950629904866, 0.1973484456539154, 0.2633756697177887, 0.34943076968193054, 0.0012102096807211637, 0.8342975974082947, 0.31870752573013306, 0.45533719658851624, 0.15729404985904694, 0.75, 0.0, 0.0, 0.0, 0.0], 'state_min': [-0.13899999856948853, -0.09099991619586945, -5.989517215532203e-11, -0.20900000631809235, -0.1589999943971634, -0.210999995470047, -0.3009999990463257, -0.01600000075995922, -0.1860000044107437, -0.6940000057220459, 0.0, 0.0, 0.0, 0.0, -0.1019991859793663, -0.06899992376565933, -0.16899999976158142, -0.14499999582767487, -0.3709999918937683, -0.28700000047683716, -0.17800045013427734, -0.4869999885559082, -0.7599999904632568, -0.27300000190734863, -0.029999999329447746, 0.0, -0.39100033044815063, -0.0010000000474974513, 0.0, -0.15000000596046448, 0.0, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.0860000029206276, 0.2720000147819519, 0.23100000619888306, 0.0, 6.510182259944486e-08, 0.0, 0.0, 0.5550000071525574, 0.02100004442036152, 0.0, 0.5429999828338623, 1.13100004196167, 0.5770000219345093, 0.9580000042915344, 0.07141251862049103, 0.10899999737739563, 0.04699999839067459, 0.13600000739097595, 0.0, 0.003000000026077032, 0.0, 0.009999999776482582, 0.0020000000949949026, 0.2029999941587448, 0.2759999930858612, 0.3499999940395355, 0.003000000026077032, 0.8370000123977661, 0.0, 0.0, 0.0, 0.75, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyBendPick-v0'], 'val_repo_ids': ['G1WholebodyBendPick-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '0,1,2,3', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '37919'}}
|
| 21 |
+
2026-03-18 19:53:48,477 INFO MainThread:280442 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/agq65opl
|
| 22 |
+
2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
|
| 23 |
+
2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_restore():2472] restore
|
| 24 |
+
2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_restore():2478] restore done
|
| 25 |
+
2026-03-18 19:53:50,040 INFO MainThread:280442 [wandb_run.py:_footer_sync_info():3868] logging synced files
|
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/run-agq65opl.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c4763659b36203ca292aadbe415653bf031cd65a551620a19105912bae817a97
|
| 3 |
+
size 52535712
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/argv.txt
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
scripts/train.py
|
| 2 |
+
simple_act_config
|
| 3 |
+
--seed=2026
|
| 4 |
+
--exp=g1wholebodyhandover-v0
|
| 5 |
+
--train.name=act-g1
|
| 6 |
+
--log.report-to=wandb
|
| 7 |
+
--train.data_parallel=ddp
|
| 8 |
+
--train.mixed_precision=bf16
|
| 9 |
+
--train.train-batch-size=32
|
| 10 |
+
--train.warmup-steps=1000
|
| 11 |
+
--train.warmup-ratio=None
|
| 12 |
+
--train.checkpointing-steps=5000
|
| 13 |
+
--train.validation_steps=500
|
| 14 |
+
--train.val_num_batches=20
|
| 15 |
+
--train.gradient_accumulation_steps=1
|
| 16 |
+
--train.max-training-steps=40000
|
| 17 |
+
--train.learning-rate=1e-4
|
| 18 |
+
--train.max-grad-norm=1.0
|
| 19 |
+
--train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 20 |
+
--train.lr_scheduler_kwargs.betas 0.95 0.999
|
| 21 |
+
--train.lr_scheduler_type=cosine
|
| 22 |
+
--data.root_dir=/data/jliu/data
|
| 23 |
+
--data.train-repo-ids=G1WholebodyHandover-v0
|
| 24 |
+
--data.transform.repack.action_chunk_size=30
|
| 25 |
+
--data.transform.repack.pad-action-dim=36
|
| 26 |
+
--data.transform.repack.pad-state-dim=36
|
| 27 |
+
--data.transform.field.stat-path=meta/stats_psi0.json
|
| 28 |
+
--data.transform.field.stat-action-key=action
|
| 29 |
+
--data.transform.field.stat-state-key=states
|
| 30 |
+
--data.transform.field.normalize-state
|
| 31 |
+
--data.transform.field.action-norm-type=bounds
|
| 32 |
+
--data.transform.field.pad-action-dim=36
|
| 33 |
+
--data.transform.field.pad-state-dim=36
|
| 34 |
+
--data.transform.model.img-aug
|
| 35 |
+
--model.chunk-size=30
|
| 36 |
+
--model.n-action-steps=30
|
| 37 |
+
--model.action-dim=36
|
| 38 |
+
--model.state-dim=36
|
| 39 |
+
--model.use-vae
|
| 40 |
+
--model.kl-weight=10.0
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:be3cee501e8e37aa7dc99358fb114794cfe604e00da6ae82e00ec88a9b429c8a
|
| 3 |
+
size 206658952
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/optimizer.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b7dc327459001762d0a2f688167d42a05a461410278eb668a60bcd532f829d01
|
| 3 |
+
size 413111371
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_0.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:debdcafc5d326bc0876e3268ce9be296d94f9cfd67a489f44ee827cacd1c0699
|
| 3 |
+
size 15473
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_1.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:08b5ccfa2fc4720532d3399cfb5bff7d3be670aa7790b0de564ede016cb0c973
|
| 3 |
+
size 15473
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_2.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:43b76b6dcb5c2116e0217a758aadd5240580ffa1cb0873287aaef42984ccad10
|
| 3 |
+
size 15473
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/scheduler.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:edd2ef4de5774723b53b76baced1d134ab2a534e3951c7a3ebfbc157468c8f72
|
| 3 |
+
size 1401
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/envs.txt
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
OMP_NUM_THREADS=32
|
| 2 |
+
HF_HOME=/data/cache
|
| 3 |
+
TORCH_HOME=/data/cache
|
| 4 |
+
HF_TOKEN=hf_...TiKa
|
| 5 |
+
HF_LEROBOT_HOME=/data/data/lerobot
|
| 6 |
+
WE_HOME=Not Set
|
| 7 |
+
DATA_HOME=/data/data
|
| 8 |
+
UV_CACHE_DIR=/data/cache
|
| 9 |
+
WANDB_API_KEY=90e...5c06
|
| 10 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
|
| 11 |
+
CUDA_VISIBLE_DEVICES=4,5,6,7
|
| 12 |
+
WORLD_SIZE=4
|
| 13 |
+
LOCAL_WORLD_SIZE=4
|
| 14 |
+
RANK=0
|
| 15 |
+
LOCAL_RANK=0
|
| 16 |
+
MASTER_ADDR=nebula99
|
| 17 |
+
MASTER_PORT=40557
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/run_config.json
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"exp": "g1wholebodyhandover-v0",
|
| 3 |
+
"seed": 2026,
|
| 4 |
+
"auto_tag_run": false,
|
| 5 |
+
"eval": false,
|
| 6 |
+
"debug": false,
|
| 7 |
+
"timestamp": "2604071503",
|
| 8 |
+
"log": {
|
| 9 |
+
"logging_dir": "logs",
|
| 10 |
+
"report_to": "wandb",
|
| 11 |
+
"log_freq": 100
|
| 12 |
+
},
|
| 13 |
+
"wandb": {
|
| 14 |
+
"project": "psi",
|
| 15 |
+
"entity": "jliu530-soochow-university",
|
| 16 |
+
"group": "act-g1",
|
| 17 |
+
"id": "26s2ml9t",
|
| 18 |
+
"name": "g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503",
|
| 19 |
+
"resume": "allow"
|
| 20 |
+
},
|
| 21 |
+
"train": {
|
| 22 |
+
"num_workers": 8,
|
| 23 |
+
"overfit_single_batch": false,
|
| 24 |
+
"name": "act-g1",
|
| 25 |
+
"resume_from_checkpoint": null,
|
| 26 |
+
"skip_resumed_steps": false,
|
| 27 |
+
"hf_token": ".hf_token",
|
| 28 |
+
"lora": false,
|
| 29 |
+
"output_dir": ".runs",
|
| 30 |
+
"gradient_accumulation_steps": 1,
|
| 31 |
+
"mixed_precision": "bf16",
|
| 32 |
+
"max_grad_norm": 1.0,
|
| 33 |
+
"optimizer_foreach": null,
|
| 34 |
+
"train_batch_size": 32,
|
| 35 |
+
"val_batch_size": 16,
|
| 36 |
+
"val_num_batches": 20,
|
| 37 |
+
"checkpointing_steps": 5000,
|
| 38 |
+
"max_checkpoints_to_keep": null,
|
| 39 |
+
"validation_steps": 500,
|
| 40 |
+
"learning_rate": 0.0001,
|
| 41 |
+
"lr_scheduler_type": "cosine",
|
| 42 |
+
"lr_scheduler_kwargs": {
|
| 43 |
+
"betas": [
|
| 44 |
+
0.95,
|
| 45 |
+
0.999
|
| 46 |
+
],
|
| 47 |
+
"weight_decay": 1e-6,
|
| 48 |
+
"eps": 1e-8
|
| 49 |
+
},
|
| 50 |
+
"scheduler_specific_kwargs": {},
|
| 51 |
+
"data_parallel": "ddp",
|
| 52 |
+
"sharding_strategy": "full-shard",
|
| 53 |
+
"deepspeed_config": "/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json",
|
| 54 |
+
"enable_gradient_checkpointing": true,
|
| 55 |
+
"enable_mixed_precision_training": true,
|
| 56 |
+
"reduce_in_full_precision": true,
|
| 57 |
+
"max_training_steps": 40000,
|
| 58 |
+
"num_train_epochs": null,
|
| 59 |
+
"warmup_steps": 1000,
|
| 60 |
+
"warmup_ratio": null
|
| 61 |
+
},
|
| 62 |
+
"data": {
|
| 63 |
+
"transform": {
|
| 64 |
+
"repack": {
|
| 65 |
+
"dataset_name": "simple",
|
| 66 |
+
"num_past_frames": 0,
|
| 67 |
+
"action_chunk_size": 30,
|
| 68 |
+
"pad_action_dim": 36,
|
| 69 |
+
"pad_state_dim": 36
|
| 70 |
+
},
|
| 71 |
+
"model": {
|
| 72 |
+
"resize": {
|
| 73 |
+
"size": [
|
| 74 |
+
256,
|
| 75 |
+
480
|
| 76 |
+
]
|
| 77 |
+
},
|
| 78 |
+
"center_crop": {
|
| 79 |
+
"size": [
|
| 80 |
+
224,
|
| 81 |
+
224
|
| 82 |
+
]
|
| 83 |
+
},
|
| 84 |
+
"color_jitter": {
|
| 85 |
+
"brightness": 0.2,
|
| 86 |
+
"contrast": [
|
| 87 |
+
0.8,
|
| 88 |
+
1.2
|
| 89 |
+
],
|
| 90 |
+
"saturation": [
|
| 91 |
+
0.8,
|
| 92 |
+
1.2
|
| 93 |
+
],
|
| 94 |
+
"hue": 0.05
|
| 95 |
+
},
|
| 96 |
+
"normalize": {
|
| 97 |
+
"mean": [
|
| 98 |
+
0.485,
|
| 99 |
+
0.456,
|
| 100 |
+
0.406
|
| 101 |
+
],
|
| 102 |
+
"std": [
|
| 103 |
+
0.229,
|
| 104 |
+
0.224,
|
| 105 |
+
0.225
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
"img_aug": true
|
| 109 |
+
},
|
| 110 |
+
"field": {
|
| 111 |
+
"stat_path": "meta/stats_psi0.json",
|
| 112 |
+
"action_norm_type": "bounds",
|
| 113 |
+
"stat_action_key": "action",
|
| 114 |
+
"stat_state_key": "states",
|
| 115 |
+
"use_norm_mask": false,
|
| 116 |
+
"action_norm_masks": [
|
| 117 |
+
true,
|
| 118 |
+
true,
|
| 119 |
+
true,
|
| 120 |
+
true,
|
| 121 |
+
true,
|
| 122 |
+
true,
|
| 123 |
+
false
|
| 124 |
+
],
|
| 125 |
+
"action_min": [
|
| 126 |
+
-0.5,
|
| 127 |
+
-1.1095792134107943e-16,
|
| 128 |
+
-1.1095792134107943e-16,
|
| 129 |
+
-1.5,
|
| 130 |
+
-1.5,
|
| 131 |
+
-1.5,
|
| 132 |
+
-1.5,
|
| 133 |
+
-0.5,
|
| 134 |
+
-0.699999988079071,
|
| 135 |
+
-0.699999988079071,
|
| 136 |
+
-2.2166350627321588e-16,
|
| 137 |
+
-2.2166350627321588e-16,
|
| 138 |
+
0.0,
|
| 139 |
+
-2.2166350627321588e-16,
|
| 140 |
+
-0.47567468881607056,
|
| 141 |
+
0.1900009959936142,
|
| 142 |
+
-0.512170135974884,
|
| 143 |
+
-0.6265152096748352,
|
| 144 |
+
-0.5008617043495178,
|
| 145 |
+
-0.8220608830451965,
|
| 146 |
+
-0.9223371148109436,
|
| 147 |
+
-0.49507391452789307,
|
| 148 |
+
-0.3437551259994507,
|
| 149 |
+
-0.6871383190155029,
|
| 150 |
+
-0.7637607455253601,
|
| 151 |
+
-0.7568023204803467,
|
| 152 |
+
-0.576077401638031,
|
| 153 |
+
-0.4588268995285034,
|
| 154 |
+
-0.13876836001873016,
|
| 155 |
+
-0.10360867530107498,
|
| 156 |
+
-0.47856518626213074,
|
| 157 |
+
0.7400000095367432,
|
| 158 |
+
-0.5,
|
| 159 |
+
-0.5,
|
| 160 |
+
-0.26161932945251465,
|
| 161 |
+
-0.06718750298023224
|
| 162 |
+
],
|
| 163 |
+
"action_max": [
|
| 164 |
+
0.5,
|
| 165 |
+
0.699999988079071,
|
| 166 |
+
0.699999988079071,
|
| 167 |
+
0.0,
|
| 168 |
+
2.2146225653890418e-16,
|
| 169 |
+
2.2146225653890418e-16,
|
| 170 |
+
2.2146225653890418e-16,
|
| 171 |
+
1.2266071310501902e-19,
|
| 172 |
+
1.1078670818917075e-16,
|
| 173 |
+
1.1078670818917075e-16,
|
| 174 |
+
1.5,
|
| 175 |
+
1.5,
|
| 176 |
+
0.6000000238418579,
|
| 177 |
+
1.5,
|
| 178 |
+
0.2472410947084427,
|
| 179 |
+
0.7092280983924866,
|
| 180 |
+
1.2571598291397095,
|
| 181 |
+
0.42311304807662964,
|
| 182 |
+
0.8564174771308899,
|
| 183 |
+
0.5002086162567139,
|
| 184 |
+
0.5172277092933655,
|
| 185 |
+
0.16140148043632507,
|
| 186 |
+
-0.1900009959936142,
|
| 187 |
+
0.5362864136695862,
|
| 188 |
+
0.5715147256851196,
|
| 189 |
+
0.5002322196960449,
|
| 190 |
+
0.566592276096344,
|
| 191 |
+
0.6392397880554199,
|
| 192 |
+
0.1580466777086258,
|
| 193 |
+
0.2233395129442215,
|
| 194 |
+
0.2582152187824249,
|
| 195 |
+
0.7400000095367432,
|
| 196 |
+
0.5,
|
| 197 |
+
0.5,
|
| 198 |
+
0.3454970121383667,
|
| 199 |
+
0.2899305522441864
|
| 200 |
+
],
|
| 201 |
+
"state_min": [
|
| 202 |
+
-0.5564982891082764,
|
| 203 |
+
-0.48307520151138306,
|
| 204 |
+
-0.0005447770818136632,
|
| 205 |
+
-0.8388738632202148,
|
| 206 |
+
-1.3970016241073608,
|
| 207 |
+
-0.8296014666557312,
|
| 208 |
+
-1.4599460363388062,
|
| 209 |
+
-0.5806806683540344,
|
| 210 |
+
-0.5149835348129272,
|
| 211 |
+
-0.6775947213172913,
|
| 212 |
+
-0.001480442238971591,
|
| 213 |
+
-0.0002713006397243589,
|
| 214 |
+
-0.000914653530344367,
|
| 215 |
+
-0.00019419840828049928,
|
| 216 |
+
-0.4206617772579193,
|
| 217 |
+
0.13972464203834534,
|
| 218 |
+
-0.546251654624939,
|
| 219 |
+
-0.5596316456794739,
|
| 220 |
+
-0.4764360189437866,
|
| 221 |
+
-0.7253566384315491,
|
| 222 |
+
-0.9443663954734802,
|
| 223 |
+
-0.4381798803806305,
|
| 224 |
+
-0.3338131606578827,
|
| 225 |
+
-0.667724072933197,
|
| 226 |
+
-0.6881827116012573,
|
| 227 |
+
-0.7544379830360413,
|
| 228 |
+
-0.5189417600631714,
|
| 229 |
+
-0.4484957158565521,
|
| 230 |
+
-0.13709338009357452,
|
| 231 |
+
-0.07360810041427612,
|
| 232 |
+
-0.4748336970806122,
|
| 233 |
+
0.7400000095367432,
|
| 234 |
+
0.0,
|
| 235 |
+
0.0,
|
| 236 |
+
0.0,
|
| 237 |
+
0.0
|
| 238 |
+
],
|
| 239 |
+
"state_max": [
|
| 240 |
+
0.43566983938217163,
|
| 241 |
+
0.3739710748195648,
|
| 242 |
+
0.6575677990913391,
|
| 243 |
+
0.004060761071741581,
|
| 244 |
+
0.0005700877518393099,
|
| 245 |
+
0.0004725759499706328,
|
| 246 |
+
0.00010080631182063371,
|
| 247 |
+
0.00001310737025050912,
|
| 248 |
+
0.21882089972496033,
|
| 249 |
+
0.0005271440604701638,
|
| 250 |
+
0.530737042427063,
|
| 251 |
+
1.4406861066818237,
|
| 252 |
+
1.4605127573013306,
|
| 253 |
+
1.4595911502838135,
|
| 254 |
+
0.2663630545139313,
|
| 255 |
+
0.657910943031311,
|
| 256 |
+
1.2515853643417358,
|
| 257 |
+
0.502498209476471,
|
| 258 |
+
0.8292973637580872,
|
| 259 |
+
0.5248894095420837,
|
| 260 |
+
0.4653257131576538,
|
| 261 |
+
0.18638382852077484,
|
| 262 |
+
-0.16696421802043915,
|
| 263 |
+
0.49318820238113403,
|
| 264 |
+
0.6363148093223572,
|
| 265 |
+
0.45773962140083313,
|
| 266 |
+
0.6238265037536621,
|
| 267 |
+
0.653800904750824,
|
| 268 |
+
0.1436084657907486,
|
| 269 |
+
0.25937986373901367,
|
| 270 |
+
0.26422709226608276,
|
| 271 |
+
0.7400000095367432,
|
| 272 |
+
0.0,
|
| 273 |
+
0.0,
|
| 274 |
+
0.0,
|
| 275 |
+
0.0
|
| 276 |
+
],
|
| 277 |
+
"normalize_state": true,
|
| 278 |
+
"pad_action_dim": 36,
|
| 279 |
+
"pad_state_dim": 36
|
| 280 |
+
}
|
| 281 |
+
},
|
| 282 |
+
"root_dir": "/data/jliu/data",
|
| 283 |
+
"train_repo_ids": [
|
| 284 |
+
"G1WholebodyHandover-v0"
|
| 285 |
+
],
|
| 286 |
+
"val_repo_ids": [
|
| 287 |
+
"G1WholebodyHandover-v0"
|
| 288 |
+
]
|
| 289 |
+
},
|
| 290 |
+
"model": {
|
| 291 |
+
"n_obs_steps": 1,
|
| 292 |
+
"chunk_size": 30,
|
| 293 |
+
"n_action_steps": 30,
|
| 294 |
+
"action_dim": 36,
|
| 295 |
+
"state_dim": 36,
|
| 296 |
+
"dim_model": 512,
|
| 297 |
+
"n_heads": 8,
|
| 298 |
+
"dim_feedforward": 3200,
|
| 299 |
+
"feedforward_activation": "relu",
|
| 300 |
+
"n_encoder_layers": 4,
|
| 301 |
+
"n_decoder_layers": 1,
|
| 302 |
+
"pre_norm": false,
|
| 303 |
+
"dropout": 0.1,
|
| 304 |
+
"use_vae": true,
|
| 305 |
+
"latent_dim": 32,
|
| 306 |
+
"n_vae_encoder_layers": 4,
|
| 307 |
+
"kl_weight": 10.0,
|
| 308 |
+
"temporal_ensemble_coeff": null
|
| 309 |
+
}
|
| 310 |
+
}
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-07 15:03:14,310 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Configure stats pid to 2236517
|
| 3 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug.log
|
| 5 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-internal.log
|
| 6 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-07 15:03:14,567 INFO MainThread:2236517 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-07 15:03:14,577 INFO MainThread:2236517 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-07 15:03:14,578 INFO MainThread:2236517 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-07 15:03:14,584 INFO MainThread:2236517 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-07 15:03:15,461 INFO MainThread:2236517 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-07 15:03:15,625 INFO MainThread:2236517 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-07 15:03:15,626 INFO MainThread:2236517 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodyhandover-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2604071503', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.5, -1.1095792134107943e-16, -1.1095792134107943e-16, -1.5, -1.5, -1.5, -1.5, -0.5, -0.699999988079071, -0.699999988079071, -2.2166350627321588e-16, -2.2166350627321588e-16, 0.0, -2.2166350627321588e-16, -0.47567468881607056, 0.1900009959936142, -0.512170135974884, -0.6265152096748352, -0.5008617043495178, -0.8220608830451965, -0.9223371148109436, -0.49507391452789307, -0.3437551259994507, -0.6871383190155029, -0.7637607455253601, -0.7568023204803467, -0.576077401638031, -0.4588268995285034, -0.13876836001873016, -0.10360867530107498, -0.47856518626213074, 0.7400000095367432, -0.5, -0.5, -0.26161932945251465, -0.06718750298023224], 'action_max': [0.5, 0.699999988079071, 0.699999988079071, 0.0, 2.2146225653890418e-16, 2.2146225653890418e-16, 2.2146225653890418e-16, 1.2266071310501902e-19, 1.1078670818917075e-16, 1.1078670818917075e-16, 1.5, 1.5, 0.6000000238418579, 1.5, 0.2472410947084427, 0.7092280983924866, 1.2571598291397095, 0.42311304807662964, 0.8564174771308899, 0.5002086162567139, 0.5172277092933655, 0.16140148043632507, -0.1900009959936142, 0.5362864136695862, 0.5715147256851196, 0.5002322196960449, 0.566592276096344, 0.6392397880554199, 0.1580466777086258, 0.2233395129442215, 0.2582152187824249, 0.7400000095367432, 0.5, 0.5, 0.3454970121383667, 0.2899305522441864], 'state_min': [-0.5564982891082764, -0.48307520151138306, -0.0005447770818136632, -0.8388738632202148, -1.3970016241073608, -0.8296014666557312, -1.4599460363388062, -0.5806806683540344, -0.5149835348129272, -0.6775947213172913, -0.001480442238971591, -0.0002713006397243589, -0.000914653530344367, -0.00019419840828049928, -0.4206617772579193, 0.13972464203834534, -0.546251654624939, -0.5596316456794739, -0.4764360189437866, -0.7253566384315491, -0.9443663954734802, -0.4381798803806305, -0.3338131606578827, -0.667724072933197, -0.6881827116012573, -0.7544379830360413, -0.5189417600631714, -0.4484957158565521, -0.13709338009357452, -0.07360810041427612, -0.4748336970806122, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.43566983938217163, 0.3739710748195648, 0.6575677990913391, 0.004060761071741581, 0.0005700877518393099, 0.0004725759499706328, 0.00010080631182063371, 1.310737025050912e-05, 0.21882089972496033, 0.0005271440604701638, 0.530737042427063, 1.4406861066818237, 1.4605127573013306, 1.4595911502838135, 0.2663630545139313, 0.657910943031311, 1.2515853643417358, 0.502498209476471, 0.8292973637580872, 0.5248894095420837, 0.4653257131576538, 0.18638382852077484, -0.16696421802043915, 0.49318820238113403, 0.6363148093223572, 0.45773962140083313, 0.6238265037536621, 0.653800904750824, 0.1436084657907486, 0.25937986373901367, 0.26422709226608276, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyHandover-v0'], 'val_repo_ids': ['G1WholebodyHandover-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '4,5,6,7', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '40557'}}
|
| 21 |
+
2026-04-07 18:06:44,913 INFO MainThread:2236517 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/26s2ml9t
|
| 22 |
+
2026-04-07 18:06:44,914 INFO MainThread:2236517 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
|
| 23 |
+
2026-04-07 18:06:44,915 INFO MainThread:2236517 [wandb_run.py:_restore():2472] restore
|
| 24 |
+
2026-04-07 18:06:44,915 INFO MainThread:2236517 [wandb_run.py:_restore():2478] restore done
|
| 25 |
+
2026-04-07 18:06:45,754 INFO MainThread:2236517 [wandb_run.py:_footer_sync_info():3868] logging synced files
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/config.yaml
ADDED
|
@@ -0,0 +1,448 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.1
|
| 4 |
+
e:
|
| 5 |
+
1ney3xhb33gta4c2m7gfcmat7y4xd0vk:
|
| 6 |
+
args:
|
| 7 |
+
- simple_act_config
|
| 8 |
+
- --seed=2026
|
| 9 |
+
- --exp=g1wholebodyhandover-v0
|
| 10 |
+
- --train.name=act-g1
|
| 11 |
+
- --log.report-to=wandb
|
| 12 |
+
- --train.data_parallel=ddp
|
| 13 |
+
- --train.mixed_precision=bf16
|
| 14 |
+
- --train.train-batch-size=32
|
| 15 |
+
- --train.warmup-steps=1000
|
| 16 |
+
- --train.warmup-ratio=None
|
| 17 |
+
- --train.checkpointing-steps=5000
|
| 18 |
+
- --train.validation_steps=500
|
| 19 |
+
- --train.val_num_batches=20
|
| 20 |
+
- --train.gradient_accumulation_steps=1
|
| 21 |
+
- --train.max-training-steps=40000
|
| 22 |
+
- --train.learning-rate=1e-4
|
| 23 |
+
- --train.max-grad-norm=1.0
|
| 24 |
+
- --train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 25 |
+
- --train.lr_scheduler_kwargs.betas
|
| 26 |
+
- "0.95"
|
| 27 |
+
- "0.999"
|
| 28 |
+
- --train.lr_scheduler_type=cosine
|
| 29 |
+
- --data.root_dir=/data/jliu/data
|
| 30 |
+
- --data.train-repo-ids=G1WholebodyHandover-v0
|
| 31 |
+
- --data.transform.repack.action_chunk_size=30
|
| 32 |
+
- --data.transform.repack.pad-action-dim=36
|
| 33 |
+
- --data.transform.repack.pad-state-dim=36
|
| 34 |
+
- --data.transform.field.stat-path=meta/stats_psi0.json
|
| 35 |
+
- --data.transform.field.stat-action-key=action
|
| 36 |
+
- --data.transform.field.stat-state-key=states
|
| 37 |
+
- --data.transform.field.normalize-state
|
| 38 |
+
- --data.transform.field.action-norm-type=bounds
|
| 39 |
+
- --data.transform.field.pad-action-dim=36
|
| 40 |
+
- --data.transform.field.pad-state-dim=36
|
| 41 |
+
- --data.transform.model.img-aug
|
| 42 |
+
- --model.chunk-size=30
|
| 43 |
+
- --model.n-action-steps=30
|
| 44 |
+
- --model.action-dim=36
|
| 45 |
+
- --model.state-dim=36
|
| 46 |
+
- --model.use-vae
|
| 47 |
+
- --model.kl-weight=10.0
|
| 48 |
+
codePath: scripts/train.py
|
| 49 |
+
codePathLocal: scripts/train.py
|
| 50 |
+
cpu_count: 128
|
| 51 |
+
cpu_count_logical: 128
|
| 52 |
+
cudaVersion: "12.9"
|
| 53 |
+
disk:
|
| 54 |
+
/:
|
| 55 |
+
total: "105089261568"
|
| 56 |
+
used: "89297772544"
|
| 57 |
+
email: jliu530@163.com
|
| 58 |
+
executable: /data/jliu/psi/.venv-psi/bin/python3
|
| 59 |
+
git:
|
| 60 |
+
commit: 04614628ecb677f5f278e2d31c1103aed8127f26
|
| 61 |
+
remote: https://github.com/songlin/psi.git
|
| 62 |
+
gpu: NVIDIA A100-SXM4-80GB
|
| 63 |
+
gpu_count: 8
|
| 64 |
+
gpu_nvidia:
|
| 65 |
+
- architecture: Ampere
|
| 66 |
+
cudaCores: 6912
|
| 67 |
+
memoryTotal: "85899345920"
|
| 68 |
+
name: NVIDIA A100-SXM4-80GB
|
| 69 |
+
uuid: GPU-02b51758-2aec-8c71-c751-92567fdb15ad
|
| 70 |
+
- architecture: Ampere
|
| 71 |
+
cudaCores: 6912
|
| 72 |
+
memoryTotal: "85899345920"
|
| 73 |
+
name: NVIDIA A100-SXM4-80GB
|
| 74 |
+
uuid: GPU-be645bb4-1ef0-15da-fc6b-53c5378650df
|
| 75 |
+
- architecture: Ampere
|
| 76 |
+
cudaCores: 6912
|
| 77 |
+
memoryTotal: "85899345920"
|
| 78 |
+
name: NVIDIA A100-SXM4-80GB
|
| 79 |
+
uuid: GPU-44868ccd-5809-61e2-f8cc-1dff8efc7eb1
|
| 80 |
+
- architecture: Ampere
|
| 81 |
+
cudaCores: 6912
|
| 82 |
+
memoryTotal: "85899345920"
|
| 83 |
+
name: NVIDIA A100-SXM4-80GB
|
| 84 |
+
uuid: GPU-a12b8de3-fb12-2901-c3a2-1e0e2b050b7b
|
| 85 |
+
- architecture: Ampere
|
| 86 |
+
cudaCores: 6912
|
| 87 |
+
memoryTotal: "85899345920"
|
| 88 |
+
name: NVIDIA A100-SXM4-80GB
|
| 89 |
+
uuid: GPU-57787183-4904-b2c9-cf52-1c1af976b365
|
| 90 |
+
- architecture: Ampere
|
| 91 |
+
cudaCores: 6912
|
| 92 |
+
memoryTotal: "85899345920"
|
| 93 |
+
name: NVIDIA A100-SXM4-80GB
|
| 94 |
+
uuid: GPU-e82d0fd0-9dcb-3d7c-9451-c4770ad6ddc3
|
| 95 |
+
- architecture: Ampere
|
| 96 |
+
cudaCores: 6912
|
| 97 |
+
memoryTotal: "85899345920"
|
| 98 |
+
name: NVIDIA A100-SXM4-80GB
|
| 99 |
+
uuid: GPU-9a825fa3-60d0-3acf-6c79-67e9230ae24c
|
| 100 |
+
- architecture: Ampere
|
| 101 |
+
cudaCores: 6912
|
| 102 |
+
memoryTotal: "85899345920"
|
| 103 |
+
name: NVIDIA A100-SXM4-80GB
|
| 104 |
+
uuid: GPU-6fe02e5e-b886-9c02-e433-6237f3897dd8
|
| 105 |
+
host: nebula99
|
| 106 |
+
memory:
|
| 107 |
+
total: "1623177445376"
|
| 108 |
+
os: Linux-6.8.0-106-generic-x86_64-with-glibc2.39
|
| 109 |
+
program: /data/jliu/psi/scripts/train.py
|
| 110 |
+
python: CPython 3.10.20
|
| 111 |
+
root: /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503
|
| 112 |
+
startedAt: "2026-04-07T15:03:14.309147Z"
|
| 113 |
+
writerId: 1ney3xhb33gta4c2m7gfcmat7y4xd0vk
|
| 114 |
+
m: []
|
| 115 |
+
python_version: 3.10.20
|
| 116 |
+
t:
|
| 117 |
+
"1":
|
| 118 |
+
- 1
|
| 119 |
+
- 11
|
| 120 |
+
- 41
|
| 121 |
+
- 49
|
| 122 |
+
- 71
|
| 123 |
+
"2":
|
| 124 |
+
- 1
|
| 125 |
+
- 11
|
| 126 |
+
- 41
|
| 127 |
+
- 49
|
| 128 |
+
- 51
|
| 129 |
+
- 71
|
| 130 |
+
- 83
|
| 131 |
+
"3":
|
| 132 |
+
- 2
|
| 133 |
+
- 13
|
| 134 |
+
- 61
|
| 135 |
+
"4": 3.10.20
|
| 136 |
+
"5": 0.25.1
|
| 137 |
+
"6": 4.57.0
|
| 138 |
+
"12": 0.25.1
|
| 139 |
+
"13": linux-x86_64
|
| 140 |
+
auto_tag_run:
|
| 141 |
+
value: false
|
| 142 |
+
data:
|
| 143 |
+
value:
|
| 144 |
+
root_dir: /data/jliu/data
|
| 145 |
+
train_repo_ids:
|
| 146 |
+
- G1WholebodyHandover-v0
|
| 147 |
+
transform:
|
| 148 |
+
field:
|
| 149 |
+
action_max:
|
| 150 |
+
- 0.5
|
| 151 |
+
- 0.699999988079071
|
| 152 |
+
- 0.699999988079071
|
| 153 |
+
- 0
|
| 154 |
+
- 2.2146225653890418e-16
|
| 155 |
+
- 2.2146225653890418e-16
|
| 156 |
+
- 2.2146225653890418e-16
|
| 157 |
+
- 1.2266071310501902e-19
|
| 158 |
+
- 1.1078670818917075e-16
|
| 159 |
+
- 1.1078670818917075e-16
|
| 160 |
+
- 1.5
|
| 161 |
+
- 1.5
|
| 162 |
+
- 0.6000000238418579
|
| 163 |
+
- 1.5
|
| 164 |
+
- 0.2472410947084427
|
| 165 |
+
- 0.7092280983924866
|
| 166 |
+
- 1.2571598291397095
|
| 167 |
+
- 0.42311304807662964
|
| 168 |
+
- 0.8564174771308899
|
| 169 |
+
- 0.5002086162567139
|
| 170 |
+
- 0.5172277092933655
|
| 171 |
+
- 0.16140148043632507
|
| 172 |
+
- -0.1900009959936142
|
| 173 |
+
- 0.5362864136695862
|
| 174 |
+
- 0.5715147256851196
|
| 175 |
+
- 0.5002322196960449
|
| 176 |
+
- 0.566592276096344
|
| 177 |
+
- 0.6392397880554199
|
| 178 |
+
- 0.1580466777086258
|
| 179 |
+
- 0.2233395129442215
|
| 180 |
+
- 0.2582152187824249
|
| 181 |
+
- 0.7400000095367432
|
| 182 |
+
- 0.5
|
| 183 |
+
- 0.5
|
| 184 |
+
- 0.3454970121383667
|
| 185 |
+
- 0.2899305522441864
|
| 186 |
+
action_min:
|
| 187 |
+
- -0.5
|
| 188 |
+
- -1.1095792134107943e-16
|
| 189 |
+
- -1.1095792134107943e-16
|
| 190 |
+
- -1.5
|
| 191 |
+
- -1.5
|
| 192 |
+
- -1.5
|
| 193 |
+
- -1.5
|
| 194 |
+
- -0.5
|
| 195 |
+
- -0.699999988079071
|
| 196 |
+
- -0.699999988079071
|
| 197 |
+
- -2.2166350627321588e-16
|
| 198 |
+
- -2.2166350627321588e-16
|
| 199 |
+
- 0
|
| 200 |
+
- -2.2166350627321588e-16
|
| 201 |
+
- -0.47567468881607056
|
| 202 |
+
- 0.1900009959936142
|
| 203 |
+
- -0.512170135974884
|
| 204 |
+
- -0.6265152096748352
|
| 205 |
+
- -0.5008617043495178
|
| 206 |
+
- -0.8220608830451965
|
| 207 |
+
- -0.9223371148109436
|
| 208 |
+
- -0.49507391452789307
|
| 209 |
+
- -0.3437551259994507
|
| 210 |
+
- -0.6871383190155029
|
| 211 |
+
- -0.7637607455253601
|
| 212 |
+
- -0.7568023204803467
|
| 213 |
+
- -0.576077401638031
|
| 214 |
+
- -0.4588268995285034
|
| 215 |
+
- -0.13876836001873016
|
| 216 |
+
- -0.10360867530107498
|
| 217 |
+
- -0.47856518626213074
|
| 218 |
+
- 0.7400000095367432
|
| 219 |
+
- -0.5
|
| 220 |
+
- -0.5
|
| 221 |
+
- -0.26161932945251465
|
| 222 |
+
- -0.06718750298023224
|
| 223 |
+
action_norm_masks:
|
| 224 |
+
- true
|
| 225 |
+
- true
|
| 226 |
+
- true
|
| 227 |
+
- true
|
| 228 |
+
- true
|
| 229 |
+
- true
|
| 230 |
+
- false
|
| 231 |
+
action_norm_type: bounds
|
| 232 |
+
normalize_state: true
|
| 233 |
+
pad_action_dim: 36
|
| 234 |
+
pad_state_dim: 36
|
| 235 |
+
stat_action_key: action
|
| 236 |
+
stat_path: meta/stats_psi0.json
|
| 237 |
+
stat_state_key: states
|
| 238 |
+
state_max:
|
| 239 |
+
- 0.43566983938217163
|
| 240 |
+
- 0.3739710748195648
|
| 241 |
+
- 0.6575677990913391
|
| 242 |
+
- 0.004060761071741581
|
| 243 |
+
- 0.0005700877518393099
|
| 244 |
+
- 0.0004725759499706328
|
| 245 |
+
- 0.00010080631182063371
|
| 246 |
+
- 1.310737025050912e-05
|
| 247 |
+
- 0.21882089972496033
|
| 248 |
+
- 0.0005271440604701638
|
| 249 |
+
- 0.530737042427063
|
| 250 |
+
- 1.4406861066818237
|
| 251 |
+
- 1.4605127573013306
|
| 252 |
+
- 1.4595911502838135
|
| 253 |
+
- 0.2663630545139313
|
| 254 |
+
- 0.657910943031311
|
| 255 |
+
- 1.2515853643417358
|
| 256 |
+
- 0.502498209476471
|
| 257 |
+
- 0.8292973637580872
|
| 258 |
+
- 0.5248894095420837
|
| 259 |
+
- 0.4653257131576538
|
| 260 |
+
- 0.18638382852077484
|
| 261 |
+
- -0.16696421802043915
|
| 262 |
+
- 0.49318820238113403
|
| 263 |
+
- 0.6363148093223572
|
| 264 |
+
- 0.45773962140083313
|
| 265 |
+
- 0.6238265037536621
|
| 266 |
+
- 0.653800904750824
|
| 267 |
+
- 0.1436084657907486
|
| 268 |
+
- 0.25937986373901367
|
| 269 |
+
- 0.26422709226608276
|
| 270 |
+
- 0.7400000095367432
|
| 271 |
+
- 0
|
| 272 |
+
- 0
|
| 273 |
+
- 0
|
| 274 |
+
- 0
|
| 275 |
+
state_min:
|
| 276 |
+
- -0.5564982891082764
|
| 277 |
+
- -0.48307520151138306
|
| 278 |
+
- -0.0005447770818136632
|
| 279 |
+
- -0.8388738632202148
|
| 280 |
+
- -1.3970016241073608
|
| 281 |
+
- -0.8296014666557312
|
| 282 |
+
- -1.4599460363388062
|
| 283 |
+
- -0.5806806683540344
|
| 284 |
+
- -0.5149835348129272
|
| 285 |
+
- -0.6775947213172913
|
| 286 |
+
- -0.001480442238971591
|
| 287 |
+
- -0.0002713006397243589
|
| 288 |
+
- -0.000914653530344367
|
| 289 |
+
- -0.00019419840828049928
|
| 290 |
+
- -0.4206617772579193
|
| 291 |
+
- 0.13972464203834534
|
| 292 |
+
- -0.546251654624939
|
| 293 |
+
- -0.5596316456794739
|
| 294 |
+
- -0.4764360189437866
|
| 295 |
+
- -0.7253566384315491
|
| 296 |
+
- -0.9443663954734802
|
| 297 |
+
- -0.4381798803806305
|
| 298 |
+
- -0.3338131606578827
|
| 299 |
+
- -0.667724072933197
|
| 300 |
+
- -0.6881827116012573
|
| 301 |
+
- -0.7544379830360413
|
| 302 |
+
- -0.5189417600631714
|
| 303 |
+
- -0.4484957158565521
|
| 304 |
+
- -0.13709338009357452
|
| 305 |
+
- -0.07360810041427612
|
| 306 |
+
- -0.4748336970806122
|
| 307 |
+
- 0.7400000095367432
|
| 308 |
+
- 0
|
| 309 |
+
- 0
|
| 310 |
+
- 0
|
| 311 |
+
- 0
|
| 312 |
+
use_norm_mask: false
|
| 313 |
+
model:
|
| 314 |
+
center_crop:
|
| 315 |
+
size:
|
| 316 |
+
- 224
|
| 317 |
+
- 224
|
| 318 |
+
color_jitter:
|
| 319 |
+
brightness: 0.2
|
| 320 |
+
contrast:
|
| 321 |
+
- 0.8
|
| 322 |
+
- 1.2
|
| 323 |
+
hue: 0.05
|
| 324 |
+
saturation:
|
| 325 |
+
- 0.8
|
| 326 |
+
- 1.2
|
| 327 |
+
img_aug: true
|
| 328 |
+
normalize:
|
| 329 |
+
mean:
|
| 330 |
+
- 0.485
|
| 331 |
+
- 0.456
|
| 332 |
+
- 0.406
|
| 333 |
+
std:
|
| 334 |
+
- 0.229
|
| 335 |
+
- 0.224
|
| 336 |
+
- 0.225
|
| 337 |
+
resize:
|
| 338 |
+
size:
|
| 339 |
+
- 256
|
| 340 |
+
- 480
|
| 341 |
+
repack:
|
| 342 |
+
action_chunk_size: 30
|
| 343 |
+
dataset_name: simple
|
| 344 |
+
num_past_frames: 0
|
| 345 |
+
pad_action_dim: 36
|
| 346 |
+
pad_state_dim: 36
|
| 347 |
+
val_repo_ids:
|
| 348 |
+
- G1WholebodyHandover-v0
|
| 349 |
+
debug:
|
| 350 |
+
value: false
|
| 351 |
+
environment_variables:
|
| 352 |
+
value:
|
| 353 |
+
CUDA_VISIBLE_DEVICES: 4,5,6,7
|
| 354 |
+
DATA_HOME: /data/data
|
| 355 |
+
HF_HOME: /data/cache
|
| 356 |
+
HF_LEROBOT_HOME: /data/data/lerobot
|
| 357 |
+
HF_TOKEN: hf_...TiKa
|
| 358 |
+
LOCAL_RANK: "0"
|
| 359 |
+
LOCAL_WORLD_SIZE: "4"
|
| 360 |
+
MASTER_ADDR: nebula99
|
| 361 |
+
MASTER_PORT: "40557"
|
| 362 |
+
OMP_NUM_THREADS: "32"
|
| 363 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
|
| 364 |
+
RANK: "0"
|
| 365 |
+
TORCH_HOME: /data/cache
|
| 366 |
+
UV_CACHE_DIR: /data/cache
|
| 367 |
+
WANDB_API_KEY: 90e...5c06
|
| 368 |
+
WE_HOME: Not Set
|
| 369 |
+
WORLD_SIZE: "4"
|
| 370 |
+
eval:
|
| 371 |
+
value: false
|
| 372 |
+
exp:
|
| 373 |
+
value: g1wholebodyhandover-v0
|
| 374 |
+
log:
|
| 375 |
+
value:
|
| 376 |
+
log_freq: 100
|
| 377 |
+
logging_dir: logs
|
| 378 |
+
report_to: wandb
|
| 379 |
+
model:
|
| 380 |
+
value:
|
| 381 |
+
action_dim: 36
|
| 382 |
+
chunk_size: 30
|
| 383 |
+
dim_feedforward: 3200
|
| 384 |
+
dim_model: 512
|
| 385 |
+
dropout: 0.1
|
| 386 |
+
feedforward_activation: relu
|
| 387 |
+
kl_weight: 10
|
| 388 |
+
latent_dim: 32
|
| 389 |
+
n_action_steps: 30
|
| 390 |
+
n_decoder_layers: 1
|
| 391 |
+
n_encoder_layers: 4
|
| 392 |
+
n_heads: 8
|
| 393 |
+
n_obs_steps: 1
|
| 394 |
+
n_vae_encoder_layers: 4
|
| 395 |
+
pre_norm: false
|
| 396 |
+
state_dim: 36
|
| 397 |
+
temporal_ensemble_coeff: null
|
| 398 |
+
use_vae: true
|
| 399 |
+
seed:
|
| 400 |
+
value: 2026
|
| 401 |
+
timestamp:
|
| 402 |
+
value: "2604071503"
|
| 403 |
+
train:
|
| 404 |
+
value:
|
| 405 |
+
checkpointing_steps: 5000
|
| 406 |
+
data_parallel: ddp
|
| 407 |
+
deepspeed_config: /data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json
|
| 408 |
+
enable_gradient_checkpointing: true
|
| 409 |
+
enable_mixed_precision_training: true
|
| 410 |
+
gradient_accumulation_steps: 1
|
| 411 |
+
hf_token: .hf_token
|
| 412 |
+
learning_rate: 0.0001
|
| 413 |
+
lora: false
|
| 414 |
+
lr_scheduler_kwargs:
|
| 415 |
+
betas:
|
| 416 |
+
- 0.95
|
| 417 |
+
- 0.999
|
| 418 |
+
eps: 1e-08
|
| 419 |
+
weight_decay: 1e-06
|
| 420 |
+
lr_scheduler_type: cosine
|
| 421 |
+
max_checkpoints_to_keep: null
|
| 422 |
+
max_grad_norm: 1
|
| 423 |
+
max_training_steps: 40000
|
| 424 |
+
mixed_precision: bf16
|
| 425 |
+
name: act-g1
|
| 426 |
+
num_train_epochs: null
|
| 427 |
+
num_workers: 8
|
| 428 |
+
optimizer_foreach: null
|
| 429 |
+
output_dir: .runs
|
| 430 |
+
overfit_single_batch: false
|
| 431 |
+
reduce_in_full_precision: true
|
| 432 |
+
resume_from_checkpoint: null
|
| 433 |
+
sharding_strategy: full-shard
|
| 434 |
+
skip_resumed_steps: false
|
| 435 |
+
train_batch_size: 32
|
| 436 |
+
val_batch_size: 16
|
| 437 |
+
val_num_batches: 20
|
| 438 |
+
validation_steps: 500
|
| 439 |
+
warmup_ratio: null
|
| 440 |
+
warmup_steps: 1000
|
| 441 |
+
wandb:
|
| 442 |
+
value:
|
| 443 |
+
entity: jliu530-soochow-university
|
| 444 |
+
group: null
|
| 445 |
+
id: null
|
| 446 |
+
name: null
|
| 447 |
+
project: psi
|
| 448 |
+
resume: allow
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/output.log
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2;36m[15:03:15 04/07][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Saved configuration to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m ]8;id=805427;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=514750;file:///data/jliu/psi/scripts/train.py#128\[2m128[0m]8;;\
|
| 2 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Training configurations: ]8;id=2657;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=644039;file:///data/jliu/psi/scripts/train.py#181\[2m181[0m]8;;\
|
| 3 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> training task: [32m'act-g1'[0m ]8;id=471497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=12046;file:///data/jliu/psi/scripts/train.py#182\[2m182[0m]8;;\
|
| 4 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> run name: g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m ]8;id=329497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=220436;file:///data/jliu/psi/scripts/train.py#183\[2m183[0m]8;;\
|
| 5 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> seed: [1;36m2026[0m ]8;id=844246;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=394625;file:///data/jliu/psi/scripts/train.py#184\[2m184[0m]8;;\
|
| 6 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=94020;file:///data/jliu/psi/scripts/train.py#185\[2m185[0m]8;;\
|
| 7 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> warmup steps: [1;36m1000[0m ]8;id=147732;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=919050;file:///data/jliu/psi/scripts/train.py#186\[2m186[0m]8;;\
|
| 8 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> validation steps: [1;36m500[0m ]8;id=25941;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=888976;file:///data/jliu/psi/scripts/train.py#187\[2m187[0m]8;;\
|
| 9 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> checkpoint steps: [1;36m5000[0m ]8;id=95089;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=418144;file:///data/jliu/psi/scripts/train.py#188\[2m188[0m]8;;\
|
| 10 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> max gradient norm: [1;36m1.0[0m ]8;id=122104;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=446109;file:///data/jliu/psi/scripts/train.py#189\[2m189[0m]8;;\
|
| 11 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> PyTorch version [1;36m2.7[0m.[1;36m0[0m available. ]8;id=514883;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py\[2mconfig.py[0m]8;;\[2m:[0m]8;id=417488;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py#54\[2m54[0m]8;;\
|
| 12 |
+
Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 170085.32it/s]
|
| 13 |
+
Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 186828.69it/s]
|
| 14 |
+
[2;36m[15:03:17 04/07][0m[2;36m [0m[34mINFO [0m | >> |=> Num training samples: ]8;id=591396;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=971225;file:///data/jliu/psi/scripts/train.py#192\[2m192[0m]8;;\
|
| 15 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Training dataset size: [1;36m45[0m,[1;36m530[0m ]8;id=603472;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=235431;file:///data/jliu/psi/scripts/train.py#193\[2m193[0m]8;;\
|
| 16 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Val dataset size: [1;36m45[0m,[1;36m530[0m ]8;id=548632;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=929903;file:///data/jliu/psi/scripts/train.py#195\[2m195[0m]8;;\
|
| 17 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Initialize optimizers and schedulers[33m...[0m ]8;id=492873;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=149330;file:///data/jliu/psi/scripts/train.py#198\[2m198[0m]8;;\
|
| 18 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m ***** Running training ***** ]8;id=127940;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=657111;file:///data/jliu/psi/scripts/train.py#202\[2m202[0m]8;;\
|
| 19 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num training examples = [1;36m45530[0m ]8;id=974614;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=572136;file:///data/jliu/psi/scripts/train.py#203\[2m203[0m]8;;\
|
| 20 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Max training Epochs = [1;36m113[0m ]8;id=574775;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=838653;file:///data/jliu/psi/scripts/train.py#204\[2m204[0m]8;;\
|
| 21 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Total optimization steps = [1;36m40000[0m ]8;id=927413;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=421473;file:///data/jliu/psi/scripts/train.py#205\[2m205[0m]8;;\
|
| 22 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num steps Per Epoch = [1;36m356[0m ]8;id=9727;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=34717;file:///data/jliu/psi/scripts/train.py#206\[2m206[0m]8;;\
|
| 23 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Effective training epochs = [1;36m449.44[0m ]8;id=842580;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=548288;file:///data/jliu/psi/scripts/train.py#207\[2m207[0m]8;;\
|
| 24 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Global train batch size [1m([0mw. parallel, distributed & accumulation[1m)[0m = [1;36m128[0m ]8;id=272745;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=726293;file:///data/jliu/psi/scripts/train.py#208\[2m208[0m]8;;\
|
| 25 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Device train batch size = [1;36m32[0m ]8;id=521307;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=847206;file:///data/jliu/psi/scripts/train.py#209\[2m209[0m]8;;\
|
| 26 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Gradient Accumulation steps = [1;36m1[0m ]8;id=871789;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=304159;file:///data/jliu/psi/scripts/train.py#210\[2m210[0m]8;;\
|
| 27 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num processes [1m([0mGPUs[1m)[0m = [1;36m4[0m ]8;id=459749;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=722818;file:///data/jliu/psi/scripts/train.py#211\[2m211[0m]8;;\
|
| 28 |
+
[2;36m[15:03:19 04/07][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Accelerator runs in: .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m ]8;id=966756;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=191940;file:///data/jliu/psi/scripts/train.py#218\[2m218[0m]8;;\
|
| 29 |
+
Traing steps: 25%|██████████████████████▏ | 9999/40000 [46:38<1:13:12, 6.83it/s, loss=0.0668, lr=8.7e-05]
|
| 30 |
+
[2026-04-07 15:03:24,386] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 31 |
+
[2;36m[15:03:24 04/07][0m[2;36m [0m[34mINFO [0m | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=680052;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=279409;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 32 |
+
[2;36m [0m [35m/tmp/tmpmobd8a5b/[0m[95mtest.c[0m -o [35m/tmp/tmpmobd8a5b/[0m[95mtest.o[0m [2m [0m
|
| 33 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread [35m/tmp/tmpmobd8a5b/[0m[95mtest.o[0m -laio -o [35m/tmp/tmpmobd8a5b/[0m[95ma.out[0m ]8;id=204038;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=783864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 34 |
+
[2;36m[15:03:25 04/07][0m[2;36m [0m[34mINFO [0m | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=960999;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=14714;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 35 |
+
[2;36m [0m [35m/tmp/tmp_8xrvwrl/[0m[95mtest.c[0m -o [35m/tmp/tmp_8xrvwrl/[0m[95mtest.o[0m [2m [0m
|
| 36 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread [35m/tmp/tmp_8xrvwrl/[0m[95mtest.o[0m -L/usr/local/cuda -L/usr/local/cuda/lib64 -lcufile -o [35m/tmp/tmp_8xrvwrl/[0m[95ma.out[0m ]8;id=573141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=977459;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 37 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=208448;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=259876;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 38 |
+
[2;36m [0m [35m/tmp/tmpawj10h7g/[0m[95mtest.c[0m -o [35m/tmp/tmpawj10h7g/[0m[95mtest.o[0m [2m [0m
|
| 39 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread [35m/tmp/tmpawj10h7g/[0m[95mtest.o[0m -laio -o [35m/tmp/tmpawj10h7g/[0m[95ma.out[0m ]8;id=572992;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=281580;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 40 |
+
[2026-04-07 15:03:25,792] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
|
| 41 |
+
|
| 42 |
+
[2;36m[15:27:09 04/07][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=328668;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=696864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 43 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_5000 [2m [0m
|
| 44 |
+
[2;36m[15:27:11 04/07][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=515887;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=723262;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 45 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_5000/model.safetensors [2m [0m
|
| 46 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=495959;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=616494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 47 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_5000/optimizer.bin [2m [0m
|
| 48 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=304800;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=88534;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 49 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_5000/scheduler.bin [2m [0m
|
| 50 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=821494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=820417;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 51 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_5000/sampler.bin [2m [0m
|
| 52 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=432054;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=409371;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 53 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_5000/sampler_1.bin [2m [0m
|
| 54 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=394271;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=14818;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 55 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_5000/random_states_0.pkl [2m [0m
|
| 56 |
+
Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_5000
|
| 57 |
+
[2;36m[15:49:59 04/07][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=53870;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=874302;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 58 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_10000 [2m [0m
|
| 59 |
+
[2;36m[15:50:00 04/07][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=806914;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=990978;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 60 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_10000/model.safetensors [2m [0m
|
| 61 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=343681;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=715961;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 62 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_10000/optimizer.bin [2m [0m
|
| 63 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=127268;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=521739;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 64 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_10000/scheduler.bin [2m [0m
|
| 65 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=844039;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=179027;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 66 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_10000/sampler.bin [2m [0m
|
| 67 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=454670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=653622;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 68 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_10000/sampler_1.bin [2m [0m
|
| 69 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=711457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=745414;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 70 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_10000/random_states_0.pkl [2m [0m
|
| 71 |
+
Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_10000
|
| 72 |
+
[2;36m[16:12:47 04/07][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=204990;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=678670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 73 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_15000 [2m [0m
|
| 74 |
+
[2;36m[16:12:48 04/07][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=571229;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=976178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 75 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_15000/model.safetensors [2m [0m
|
| 76 |
+
[2;36m[16:12:49 04/07][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=791370;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=671982;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 77 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_15000/optimizer.bin [2m [0m
|
| 78 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=920457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=21464;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 79 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_15000/scheduler.bin [2m [0m
|
| 80 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=823560;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=925542;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 81 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_15000/sampler.bin [2m [0m
|
| 82 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=693342;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=885391;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 83 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_15000/sampler_1.bin [2m [0m
|
| 84 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=558902;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=434201;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 85 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_15000/random_states_0.pkl [2m [0m
|
| 86 |
+
Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_15000
|
| 87 |
+
[2;36m[16:35:30 04/07][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=196292;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=360779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 88 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_20000 [2m [0m
|
| 89 |
+
[2;36m[16:35:31 04/07][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=818149;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=854583;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 90 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_20000/model.safetensors [2m [0m
|
| 91 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=341614;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=544562;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 92 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_20000/optimizer.bin [2m [0m
|
| 93 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=835298;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=550186;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 94 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_20000/scheduler.bin [2m [0m
|
| 95 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=889056;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=224742;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 96 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_20000/sampler.bin [2m [0m
|
| 97 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=921300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=476172;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 98 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_20000/sampler_1.bin [2m [0m
|
| 99 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=821364;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=874045;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 100 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_20000/random_states_0.pkl [2m [0m
|
| 101 |
+
Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_20000
|
| 102 |
+
[2;36m[16:58:18 04/07][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=28505;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=369046;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 103 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_25000 [2m [0m
|
| 104 |
+
[2;36m[16:58:19 04/07][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=720536;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=600133;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 105 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_25000/model.safetensors [2m [0m
|
| 106 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=650939;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=750793;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 107 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_25000/optimizer.bin [2m [0m
|
| 108 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=766478;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=224851;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 109 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_25000/scheduler.bin [2m [0m
|
| 110 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=702868;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=582729;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 111 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_25000/sampler.bin [2m [0m
|
| 112 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=101114;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=457234;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 113 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_25000/sampler_1.bin [2m [0m
|
| 114 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=795987;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=373558;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 115 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_25000/random_states_0.pkl [2m [0m
|
| 116 |
+
Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_25000
|
| 117 |
+
[2;36m[17:21:02 04/07][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=660693;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=111484;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 118 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_30000 [2m [0m
|
| 119 |
+
[2;36m[17:21:03 04/07][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=878068;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=67019;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 120 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_30000/model.safetensors [2m [0m
|
| 121 |
+
[2;36m[17:21:04 04/07][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=273529;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=268565;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 122 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_30000/optimizer.bin [2m [0m
|
| 123 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=941677;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=948866;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 124 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_30000/scheduler.bin [2m [0m
|
| 125 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=60095;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=101135;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 126 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_30000/sampler.bin [2m [0m
|
| 127 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=98429;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=719141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 128 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_30000/sampler_1.bin [2m [0m
|
| 129 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=367323;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=330274;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 130 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_30000/random_states_0.pkl [2m [0m
|
| 131 |
+
Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_30000
|
| 132 |
+
[2;36m[17:43:46 04/07][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=686520;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=748910;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 133 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_35000 [2m [0m
|
| 134 |
+
[2;36m[17:43:47 04/07][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=331416;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=637309;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 135 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_35000/model.safetensors [2m [0m
|
| 136 |
+
[2;36m[17:43:48 04/07][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=708752;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=921072;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 137 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_35000/optimizer.bin [2m [0m
|
| 138 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=525611;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=714503;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 139 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_35000/scheduler.bin [2m [0m
|
| 140 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=993068;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=856493;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 141 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_35000/sampler.bin [2m [0m
|
| 142 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=457956;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=921087;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 143 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_35000/sampler_1.bin [2m [0m
|
| 144 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=303621;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=762447;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 145 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_35000/random_states_0.pkl [2m [0m
|
| 146 |
+
Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_35000
|
| 147 |
+
[2;36m[18:06:36 04/07][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=311366;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=805194;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 148 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_40000 [2m [0m
|
| 149 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=21254;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=568492;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 150 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_40000/model.safetensors [2m [0m
|
| 151 |
+
[2;36m[18:06:37 04/07][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=756734;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=118067;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 152 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_40000/optimizer.bin [2m [0m
|
| 153 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=39131;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=131787;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 154 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_40000/scheduler.bin [2m [0m
|
| 155 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=687655;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=316922;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 156 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_40000/sampler.bin [2m [0m
|
| 157 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=373263;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=435471;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 158 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_40000/sampler_1.bin [2m [0m
|
| 159 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=441435;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=694779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 160 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_40000/random_states_0.pkl [2m [0m
|
| 161 |
+
Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000
|
| 162 |
+
Training has reached maximum steps.
|
| 163 |
+
[2;36m[18:06:43 04/07][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=569300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=29797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 164 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_39999 [2m [0m
|
| 165 |
+
[2;36m[18:06:44 04/07][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=716966;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=581852;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 166 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_39999/model.safetensors [2m [0m
|
| 167 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=154857;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=239343;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 168 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_39999/optimizer.bin [2m [0m
|
| 169 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=502103;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=438178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 170 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_39999/scheduler.bin [2m [0m
|
| 171 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=657224;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=622487;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 172 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_39999/sampler.bin [2m [0m
|
| 173 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=39855;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=681797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 174 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_39999/sampler_1.bin [2m [0m
|
| 175 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=278106;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=357263;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 176 |
+
[2;36m [0m .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604071503[0m/checkpoints/ckpt_39999/random_states_0.pkl [2m [0m
|
| 177 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Finalized ACT Trainer. Epoch losses: [1m[[0m[1;36m0.0[0m, [1;36m2.2404654026031494[0m, [1;36m1.3566347360610962[0m, [1;36m0.6565650701522827[0m, [1;36m0.3426889479160309[0m, ]8;id=58988;file:///data/jliu/psi/src/psi/trainers/act_g1.py\[2mact_g1.py[0m]8;;\[2m:[0m]8;id=689824;file:///data/jliu/psi/src/psi/trainers/act_g1.py#331\[2m331[0m]8;;\
|
| 178 |
+
[2;36m [0m [1;36m0.2243376225233078[0m[1m][0m [2m [0m
|
| 179 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Happy Ending! ]8;id=61128;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=232614;file:///data/jliu/psi/scripts/train.py#310\[2m310[0m]8;;\
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/requirements.txt
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
tifffile==2025.5.10
|
| 2 |
+
mpmath==1.3.0
|
| 3 |
+
pynput==1.8.1
|
| 4 |
+
exceptiongroup==1.3.0
|
| 5 |
+
sympy==1.14.0
|
| 6 |
+
decorator==5.2.1
|
| 7 |
+
aiohttp-cors==0.8.1
|
| 8 |
+
soupsieve==2.8
|
| 9 |
+
Farama-Notifications==0.0.4
|
| 10 |
+
numpydantic==1.6.7
|
| 11 |
+
uvicorn==0.38.0
|
| 12 |
+
waterbear==2.6.8
|
| 13 |
+
comm==0.2.3
|
| 14 |
+
cmake==4.2.3
|
| 15 |
+
xxhash==3.5.0
|
| 16 |
+
parso==0.8.6
|
| 17 |
+
nvidia-cusparse-cu12==12.5.4.2
|
| 18 |
+
annotated-doc==0.0.4
|
| 19 |
+
jsonlines==4.0.0
|
| 20 |
+
nvidia-cuda-runtime-cu12==12.6.77
|
| 21 |
+
pydantic==2.10.6
|
| 22 |
+
multidict==6.6.4
|
| 23 |
+
shtab==1.7.2
|
| 24 |
+
nvidia-cuda-nvrtc-cu12==12.6.77
|
| 25 |
+
matplotlib-inline==0.2.1
|
| 26 |
+
pycollada==0.9.2
|
| 27 |
+
pure_eval==0.2.3
|
| 28 |
+
beautifulsoup4==4.14.2
|
| 29 |
+
msgspec==0.19.0
|
| 30 |
+
huggingface-hub==0.35.3
|
| 31 |
+
pydantic_core==2.27.2
|
| 32 |
+
pytz==2025.2
|
| 33 |
+
gymnasium==1.2.3
|
| 34 |
+
attrs==25.3.0
|
| 35 |
+
requests==2.32.5
|
| 36 |
+
hf_transfer==0.1.9
|
| 37 |
+
einx==0.3.0
|
| 38 |
+
scipy==1.15.3
|
| 39 |
+
transforms3d==0.4.2
|
| 40 |
+
pyzmq==27.1.0
|
| 41 |
+
aiohappyeyeballs==2.6.1
|
| 42 |
+
six==1.17.0
|
| 43 |
+
expandvars==1.1.2
|
| 44 |
+
prompt_toolkit==3.0.52
|
| 45 |
+
wcwidth==0.6.0
|
| 46 |
+
stack-data==0.6.3
|
| 47 |
+
ruamel.yaml.clib==0.2.14
|
| 48 |
+
multiprocess==0.70.16
|
| 49 |
+
vhacdx==0.0.9
|
| 50 |
+
tokenizers==0.22.2
|
| 51 |
+
aiosignal==1.4.0
|
| 52 |
+
itsdangerous==2.2.0
|
| 53 |
+
torchvision==0.22.0
|
| 54 |
+
nvidia-cublas-cu12==12.6.4.1
|
| 55 |
+
httpx==0.28.1
|
| 56 |
+
platformdirs==4.4.0
|
| 57 |
+
sentry-sdk==2.39.0
|
| 58 |
+
plotly==6.2.0
|
| 59 |
+
nvidia-cudnn-cu12==9.5.1.17
|
| 60 |
+
jsonschema==4.25.1
|
| 61 |
+
fastapi==0.119.1
|
| 62 |
+
fsspec==2025.3.0
|
| 63 |
+
MarkupSafe==3.0.3
|
| 64 |
+
pydantic-yaml==1.6.0
|
| 65 |
+
h11==0.16.0
|
| 66 |
+
typer==0.24.1
|
| 67 |
+
mypy_extensions==1.1.0
|
| 68 |
+
nest-asyncio==1.6.0
|
| 69 |
+
h5py==3.14.0
|
| 70 |
+
python-xlib==0.33
|
| 71 |
+
lazy_loader==0.4
|
| 72 |
+
einops==0.8.1
|
| 73 |
+
albumentations==1.4.18
|
| 74 |
+
params_proto==2.13.2
|
| 75 |
+
psutil==7.1.0
|
| 76 |
+
starlette==0.48.0
|
| 77 |
+
anyio==4.11.0
|
| 78 |
+
gdown==5.2.0
|
| 79 |
+
charset-normalizer==3.4.3
|
| 80 |
+
tyro==0.9.32
|
| 81 |
+
filelock==3.19.1
|
| 82 |
+
websockets==15.0.1
|
| 83 |
+
nvidia-cuda-cupti-cu12==12.6.80
|
| 84 |
+
python-dotenv==1.2.1
|
| 85 |
+
orderly-set==5.5.0
|
| 86 |
+
inquirerpy==0.3.4
|
| 87 |
+
pexpect==4.9.0
|
| 88 |
+
ipython==8.38.0
|
| 89 |
+
urllib3==2.5.0
|
| 90 |
+
diffusers==0.37.0
|
| 91 |
+
PyYAML==6.0.3
|
| 92 |
+
antlr4-python3-runtime==4.9.3
|
| 93 |
+
mdurl==0.1.2
|
| 94 |
+
omegaconf==2.3.0
|
| 95 |
+
rerun-sdk==0.22.1
|
| 96 |
+
draccus==0.10.0
|
| 97 |
+
sentencepiece==0.2.1
|
| 98 |
+
referencing==0.37.0
|
| 99 |
+
docstring_parser==0.17.0
|
| 100 |
+
protobuf==6.33.5
|
| 101 |
+
wandb==0.25.1
|
| 102 |
+
numpy==1.26.4
|
| 103 |
+
GitPython==3.1.45
|
| 104 |
+
opencv-python-headless==4.11.0.86
|
| 105 |
+
yourdfpy==0.0.58
|
| 106 |
+
async-timeout==4.0.3
|
| 107 |
+
shapely==2.1.2
|
| 108 |
+
frozenlist==1.7.0
|
| 109 |
+
simplejpeg==1.9.0
|
| 110 |
+
asttokens==3.0.1
|
| 111 |
+
Pygments==2.19.2
|
| 112 |
+
py-cpuinfo==9.0.0
|
| 113 |
+
rtree==1.4.1
|
| 114 |
+
jedi==0.19.2
|
| 115 |
+
gitdb==4.0.12
|
| 116 |
+
cloudpickle==3.1.1
|
| 117 |
+
deepspeed==0.17.1
|
| 118 |
+
executing==2.2.1
|
| 119 |
+
nvidia-nvjitlink-cu12==12.6.85
|
| 120 |
+
colorlog==6.10.1
|
| 121 |
+
qwen-vl-utils==0.0.14
|
| 122 |
+
Werkzeug==3.1.6
|
| 123 |
+
zipp==3.23.0
|
| 124 |
+
setuptools==80.9.0
|
| 125 |
+
albucore==0.0.17
|
| 126 |
+
transformers==4.57.0
|
| 127 |
+
imageio==2.34.2
|
| 128 |
+
frozendict==2.4.6
|
| 129 |
+
hjson==3.1.0
|
| 130 |
+
jsonschema-specifications==2025.9.1
|
| 131 |
+
rpds-py==0.28.0
|
| 132 |
+
tornado==6.5.5
|
| 133 |
+
blinker==1.9.0
|
| 134 |
+
accelerate==1.7.0
|
| 135 |
+
argparse==1.4.0
|
| 136 |
+
msgpack==1.1.1
|
| 137 |
+
smmap==5.0.2
|
| 138 |
+
nvidia-nvtx-cu12==12.6.77
|
| 139 |
+
packaging==25.0
|
| 140 |
+
embreex==2.17.7.post7
|
| 141 |
+
vuer==0.0.68
|
| 142 |
+
propcache==0.3.2
|
| 143 |
+
nvidia-cusolver-cu12==11.7.1.2
|
| 144 |
+
triton==3.3.0
|
| 145 |
+
pfzy==0.3.4
|
| 146 |
+
debugpy==1.8.20
|
| 147 |
+
torchcodec==0.4.0
|
| 148 |
+
dm-tree==0.1.8
|
| 149 |
+
ninja==1.13.0
|
| 150 |
+
ipykernel==7.2.0
|
| 151 |
+
traitlets==5.14.3
|
| 152 |
+
nvidia-cufile-cu12==1.11.1.6
|
| 153 |
+
datasets==3.6.0
|
| 154 |
+
nvidia-curand-cu12==10.3.7.77
|
| 155 |
+
flash_attn==2.7.4.post1
|
| 156 |
+
av==16.0.1
|
| 157 |
+
sniffio==1.3.1
|
| 158 |
+
Jinja2==3.1.6
|
| 159 |
+
narwhals==2.6.0
|
| 160 |
+
yarl==1.20.1
|
| 161 |
+
typeguard==4.4.4
|
| 162 |
+
termcolor==3.1.0
|
| 163 |
+
jupyter_client==8.8.0
|
| 164 |
+
importlib_metadata==8.7.1
|
| 165 |
+
eval_type_backport==0.2.2
|
| 166 |
+
certifi==2025.8.3
|
| 167 |
+
tzdata==2025.2
|
| 168 |
+
typing_extensions==4.15.0
|
| 169 |
+
viser==1.0.15
|
| 170 |
+
typing-inspect==0.9.0
|
| 171 |
+
annotated-types==0.7.0
|
| 172 |
+
svg.path==7.0
|
| 173 |
+
mergedeep==1.3.4
|
| 174 |
+
ptyprocess==0.7.0
|
| 175 |
+
psi==0.0.0
|
| 176 |
+
deepdiff==8.6.1
|
| 177 |
+
toml==0.10.2
|
| 178 |
+
click==8.3.0
|
| 179 |
+
dotenv==0.9.9
|
| 180 |
+
trimesh==4.8.3
|
| 181 |
+
nvidia-nccl-cu12==2.26.2
|
| 182 |
+
pyyaml-include==1.4.1
|
| 183 |
+
scikit-image==0.25.2
|
| 184 |
+
PySocks==1.7.1
|
| 185 |
+
aiohttp==3.10.5
|
| 186 |
+
opencv-python==4.11.0.86
|
| 187 |
+
nodeenv==1.9.1
|
| 188 |
+
lerobot==0.3.3
|
| 189 |
+
pillow==11.3.0
|
| 190 |
+
idna==3.10
|
| 191 |
+
Flask==3.1.3
|
| 192 |
+
killport==1.2.0
|
| 193 |
+
hf-xet==1.1.10
|
| 194 |
+
imageio-ffmpeg==0.6.0
|
| 195 |
+
httpcore==1.0.9
|
| 196 |
+
nvidia-cusparselt-cu12==0.6.3
|
| 197 |
+
networkx==3.4.2
|
| 198 |
+
ruamel.yaml==0.18.15
|
| 199 |
+
manifold3d==3.2.1
|
| 200 |
+
shellingham==1.5.4
|
| 201 |
+
nvidia-cufft-cu12==11.3.0.4
|
| 202 |
+
mapbox_earcut==1.0.3
|
| 203 |
+
peft==0.17.1
|
| 204 |
+
pyarrow==21.0.0
|
| 205 |
+
pyserial==3.5
|
| 206 |
+
torch==2.7.0
|
| 207 |
+
dill==0.3.8
|
| 208 |
+
markdown-it-py==4.0.0
|
| 209 |
+
tqdm==4.67.1
|
| 210 |
+
argcomplete==3.6.2
|
| 211 |
+
jupyter_core==5.9.1
|
| 212 |
+
pandas==2.3.3
|
| 213 |
+
lxml==6.0.2
|
| 214 |
+
safetensors==0.6.2
|
| 215 |
+
python-dateutil==2.9.0.post0
|
| 216 |
+
regex==2025.9.18
|
| 217 |
+
evdev==1.9.3
|
| 218 |
+
rich==14.1.0
|
| 219 |
+
psi==0.0.0
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-106-generic-x86_64-with-glibc2.39",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-04-07T15:03:14.309147Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"simple_act_config",
|
| 7 |
+
"--seed=2026",
|
| 8 |
+
"--exp=g1wholebodyhandover-v0",
|
| 9 |
+
"--train.name=act-g1",
|
| 10 |
+
"--log.report-to=wandb",
|
| 11 |
+
"--train.data_parallel=ddp",
|
| 12 |
+
"--train.mixed_precision=bf16",
|
| 13 |
+
"--train.train-batch-size=32",
|
| 14 |
+
"--train.warmup-steps=1000",
|
| 15 |
+
"--train.warmup-ratio=None",
|
| 16 |
+
"--train.checkpointing-steps=5000",
|
| 17 |
+
"--train.validation_steps=500",
|
| 18 |
+
"--train.val_num_batches=20",
|
| 19 |
+
"--train.gradient_accumulation_steps=1",
|
| 20 |
+
"--train.max-training-steps=40000",
|
| 21 |
+
"--train.learning-rate=1e-4",
|
| 22 |
+
"--train.max-grad-norm=1.0",
|
| 23 |
+
"--train.lr_scheduler_kwargs.weight_decay=1e-6",
|
| 24 |
+
"--train.lr_scheduler_kwargs.betas",
|
| 25 |
+
"0.95",
|
| 26 |
+
"0.999",
|
| 27 |
+
"--train.lr_scheduler_type=cosine",
|
| 28 |
+
"--data.root_dir=/data/jliu/data",
|
| 29 |
+
"--data.train-repo-ids=G1WholebodyHandover-v0",
|
| 30 |
+
"--data.transform.repack.action_chunk_size=30",
|
| 31 |
+
"--data.transform.repack.pad-action-dim=36",
|
| 32 |
+
"--data.transform.repack.pad-state-dim=36",
|
| 33 |
+
"--data.transform.field.stat-path=meta/stats_psi0.json",
|
| 34 |
+
"--data.transform.field.stat-action-key=action",
|
| 35 |
+
"--data.transform.field.stat-state-key=states",
|
| 36 |
+
"--data.transform.field.normalize-state",
|
| 37 |
+
"--data.transform.field.action-norm-type=bounds",
|
| 38 |
+
"--data.transform.field.pad-action-dim=36",
|
| 39 |
+
"--data.transform.field.pad-state-dim=36",
|
| 40 |
+
"--data.transform.model.img-aug",
|
| 41 |
+
"--model.chunk-size=30",
|
| 42 |
+
"--model.n-action-steps=30",
|
| 43 |
+
"--model.action-dim=36",
|
| 44 |
+
"--model.state-dim=36",
|
| 45 |
+
"--model.use-vae",
|
| 46 |
+
"--model.kl-weight=10.0"
|
| 47 |
+
],
|
| 48 |
+
"program": "/data/jliu/psi/scripts/train.py",
|
| 49 |
+
"codePath": "scripts/train.py",
|
| 50 |
+
"codePathLocal": "scripts/train.py",
|
| 51 |
+
"git": {
|
| 52 |
+
"remote": "https://github.com/songlin/psi.git",
|
| 53 |
+
"commit": "04614628ecb677f5f278e2d31c1103aed8127f26"
|
| 54 |
+
},
|
| 55 |
+
"email": "jliu530@163.com",
|
| 56 |
+
"root": "/data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503",
|
| 57 |
+
"host": "nebula99",
|
| 58 |
+
"executable": "/data/jliu/psi/.venv-psi/bin/python3",
|
| 59 |
+
"cpu_count": 128,
|
| 60 |
+
"cpu_count_logical": 128,
|
| 61 |
+
"gpu": "NVIDIA A100-SXM4-80GB",
|
| 62 |
+
"gpu_count": 8,
|
| 63 |
+
"disk": {
|
| 64 |
+
"/": {
|
| 65 |
+
"total": "105089261568",
|
| 66 |
+
"used": "89297772544"
|
| 67 |
+
}
|
| 68 |
+
},
|
| 69 |
+
"memory": {
|
| 70 |
+
"total": "1623177445376"
|
| 71 |
+
},
|
| 72 |
+
"gpu_nvidia": [
|
| 73 |
+
{
|
| 74 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 75 |
+
"memoryTotal": "85899345920",
|
| 76 |
+
"cudaCores": 6912,
|
| 77 |
+
"architecture": "Ampere",
|
| 78 |
+
"uuid": "GPU-02b51758-2aec-8c71-c751-92567fdb15ad"
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 82 |
+
"memoryTotal": "85899345920",
|
| 83 |
+
"cudaCores": 6912,
|
| 84 |
+
"architecture": "Ampere",
|
| 85 |
+
"uuid": "GPU-be645bb4-1ef0-15da-fc6b-53c5378650df"
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 89 |
+
"memoryTotal": "85899345920",
|
| 90 |
+
"cudaCores": 6912,
|
| 91 |
+
"architecture": "Ampere",
|
| 92 |
+
"uuid": "GPU-44868ccd-5809-61e2-f8cc-1dff8efc7eb1"
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 96 |
+
"memoryTotal": "85899345920",
|
| 97 |
+
"cudaCores": 6912,
|
| 98 |
+
"architecture": "Ampere",
|
| 99 |
+
"uuid": "GPU-a12b8de3-fb12-2901-c3a2-1e0e2b050b7b"
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 103 |
+
"memoryTotal": "85899345920",
|
| 104 |
+
"cudaCores": 6912,
|
| 105 |
+
"architecture": "Ampere",
|
| 106 |
+
"uuid": "GPU-57787183-4904-b2c9-cf52-1c1af976b365"
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 110 |
+
"memoryTotal": "85899345920",
|
| 111 |
+
"cudaCores": 6912,
|
| 112 |
+
"architecture": "Ampere",
|
| 113 |
+
"uuid": "GPU-e82d0fd0-9dcb-3d7c-9451-c4770ad6ddc3"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 117 |
+
"memoryTotal": "85899345920",
|
| 118 |
+
"cudaCores": 6912,
|
| 119 |
+
"architecture": "Ampere",
|
| 120 |
+
"uuid": "GPU-9a825fa3-60d0-3acf-6c79-67e9230ae24c"
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 124 |
+
"memoryTotal": "85899345920",
|
| 125 |
+
"cudaCores": 6912,
|
| 126 |
+
"architecture": "Ampere",
|
| 127 |
+
"uuid": "GPU-6fe02e5e-b886-9c02-e433-6237f3897dd8"
|
| 128 |
+
}
|
| 129 |
+
],
|
| 130 |
+
"cudaVersion": "12.9",
|
| 131 |
+
"writerId": "1ney3xhb33gta4c2m7gfcmat7y4xd0vk"
|
| 132 |
+
}
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"train/epoch":112,"val/denorm_err_l1_height":0,"val/denorm_err_l1_torso_target_yaw":0.00029178752447478473,"val/denorm_err_l1_rpy":0.002766967751085758,"val/denorm_err_l1_torso_vy":0.0009157538297586143,"val/denorm_err_l1_arm_joints":0.020835014060139656,"val/denorm_err_l1_torso_vyaw":0.0029068603180348873,"val/denorm_err_l1_torso_vx":0.0011786402901634574,"val/bc_loss":-0.0009642460499890149,"val/denorm_err_l1_hand_joints":0.0035619293339550495,"_step":40000,"val/kld_loss":-0.0008920701220631599,"_wandb":{"runtime":11009},"train/loss":0.00894236471503973,"_timestamp":1.7755852034701765e+09,"train/kld_loss":1.95428729057312e-05,"train/l1_loss":0.008746935985982418,"_runtime":11009.454112423}
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-core.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-07T15:03:14.405213784Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9da_xq2v/port-2236517.txt","pid":2236517,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-07T15:03:14.405914689Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2236517-2462798-4161116888/socket","Net":"unix"}}
|
| 3 |
+
{"time":"2026-04-07T15:03:14.40607627Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2236517}
|
| 4 |
+
{"time":"2026-04-07T15:03:14.567096386Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-07T15:03:14.579495643Z","level":"INFO","msg":"handleInformInit: received","streamId":"26s2ml9t","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-07T15:03:14.911097462Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"26s2ml9t","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-07T15:03:20.630060907Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fx3xgud82qm3"}
|
| 8 |
+
{"time":"2026-04-07T18:06:44.915316242Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fx3xgud82qm3"}
|
| 9 |
+
{"time":"2026-04-07T18:06:45.754451187Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fx3xgud82qm3"}
|
| 10 |
+
{"time":"2026-04-07T18:06:45.755226903Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"26s2ml9t","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-07T18:06:45.755767328Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"26s2ml9t","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-07T18:07:47.115746568Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 13 |
+
{"time":"2026-04-07T18:07:47.115798628Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-04-07T18:07:47.115808058Z","level":"INFO","msg":"server is shutting down"}
|
| 15 |
+
{"time":"2026-04-07T18:07:47.115888389Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2236517-2462798-4161116888/socket","Net":"unix"}}
|
| 16 |
+
{"time":"2026-04-07T18:07:47.115927399Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 17 |
+
{"time":"2026-04-07T18:07:47.11596726Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 18 |
+
{"time":"2026-04-07T18:07:47.11597218Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 19 |
+
{"time":"2026-04-07T18:07:47.11597774Z","level":"INFO","msg":"server is closed"}
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-07 15:03:14,310 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Configure stats pid to 2236517
|
| 3 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug.log
|
| 5 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-internal.log
|
| 6 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-07 15:03:14,567 INFO MainThread:2236517 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-07 15:03:14,577 INFO MainThread:2236517 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-07 15:03:14,578 INFO MainThread:2236517 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-07 15:03:14,584 INFO MainThread:2236517 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-07 15:03:15,461 INFO MainThread:2236517 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-07 15:03:15,625 INFO MainThread:2236517 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-07 15:03:15,626 INFO MainThread:2236517 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodyhandover-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2604071503', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.5, -1.1095792134107943e-16, -1.1095792134107943e-16, -1.5, -1.5, -1.5, -1.5, -0.5, -0.699999988079071, -0.699999988079071, -2.2166350627321588e-16, -2.2166350627321588e-16, 0.0, -2.2166350627321588e-16, -0.47567468881607056, 0.1900009959936142, -0.512170135974884, -0.6265152096748352, -0.5008617043495178, -0.8220608830451965, -0.9223371148109436, -0.49507391452789307, -0.3437551259994507, -0.6871383190155029, -0.7637607455253601, -0.7568023204803467, -0.576077401638031, -0.4588268995285034, -0.13876836001873016, -0.10360867530107498, -0.47856518626213074, 0.7400000095367432, -0.5, -0.5, -0.26161932945251465, -0.06718750298023224], 'action_max': [0.5, 0.699999988079071, 0.699999988079071, 0.0, 2.2146225653890418e-16, 2.2146225653890418e-16, 2.2146225653890418e-16, 1.2266071310501902e-19, 1.1078670818917075e-16, 1.1078670818917075e-16, 1.5, 1.5, 0.6000000238418579, 1.5, 0.2472410947084427, 0.7092280983924866, 1.2571598291397095, 0.42311304807662964, 0.8564174771308899, 0.5002086162567139, 0.5172277092933655, 0.16140148043632507, -0.1900009959936142, 0.5362864136695862, 0.5715147256851196, 0.5002322196960449, 0.566592276096344, 0.6392397880554199, 0.1580466777086258, 0.2233395129442215, 0.2582152187824249, 0.7400000095367432, 0.5, 0.5, 0.3454970121383667, 0.2899305522441864], 'state_min': [-0.5564982891082764, -0.48307520151138306, -0.0005447770818136632, -0.8388738632202148, -1.3970016241073608, -0.8296014666557312, -1.4599460363388062, -0.5806806683540344, -0.5149835348129272, -0.6775947213172913, -0.001480442238971591, -0.0002713006397243589, -0.000914653530344367, -0.00019419840828049928, -0.4206617772579193, 0.13972464203834534, -0.546251654624939, -0.5596316456794739, -0.4764360189437866, -0.7253566384315491, -0.9443663954734802, -0.4381798803806305, -0.3338131606578827, -0.667724072933197, -0.6881827116012573, -0.7544379830360413, -0.5189417600631714, -0.4484957158565521, -0.13709338009357452, -0.07360810041427612, -0.4748336970806122, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.43566983938217163, 0.3739710748195648, 0.6575677990913391, 0.004060761071741581, 0.0005700877518393099, 0.0004725759499706328, 0.00010080631182063371, 1.310737025050912e-05, 0.21882089972496033, 0.0005271440604701638, 0.530737042427063, 1.4406861066818237, 1.4605127573013306, 1.4595911502838135, 0.2663630545139313, 0.657910943031311, 1.2515853643417358, 0.502498209476471, 0.8292973637580872, 0.5248894095420837, 0.4653257131576538, 0.18638382852077484, -0.16696421802043915, 0.49318820238113403, 0.6363148093223572, 0.45773962140083313, 0.6238265037536621, 0.653800904750824, 0.1436084657907486, 0.25937986373901367, 0.26422709226608276, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyHandover-v0'], 'val_repo_ids': ['G1WholebodyHandover-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '4,5,6,7', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '40557'}}
|
| 21 |
+
2026-04-07 18:06:44,913 INFO MainThread:2236517 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/26s2ml9t
|
| 22 |
+
2026-04-07 18:06:44,914 INFO MainThread:2236517 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
|
| 23 |
+
2026-04-07 18:06:44,915 INFO MainThread:2236517 [wandb_run.py:_restore():2472] restore
|
| 24 |
+
2026-04-07 18:06:44,915 INFO MainThread:2236517 [wandb_run.py:_restore():2478] restore done
|
| 25 |
+
2026-04-07 18:06:45,754 INFO MainThread:2236517 [wandb_run.py:_footer_sync_info():3868] logging synced files
|
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/run-26s2ml9t.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9677d373b978274325f092362db630268d9300af1974e711763c57ac318d130e
|
| 3 |
+
size 47853675
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/argv.txt
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
scripts/train.py
|
| 2 |
+
simple_act_config
|
| 3 |
+
--seed=2026
|
| 4 |
+
--exp=g1wholebodylocomotionpickbetweentablesteleop-v0
|
| 5 |
+
--train.name=act-g1
|
| 6 |
+
--log.report-to=wandb
|
| 7 |
+
--train.data_parallel=ddp
|
| 8 |
+
--train.mixed_precision=bf16
|
| 9 |
+
--train.train-batch-size=32
|
| 10 |
+
--train.warmup-steps=1000
|
| 11 |
+
--train.warmup-ratio=None
|
| 12 |
+
--train.checkpointing-steps=5000
|
| 13 |
+
--train.validation_steps=500
|
| 14 |
+
--train.val_num_batches=20
|
| 15 |
+
--train.gradient_accumulation_steps=1
|
| 16 |
+
--train.max-training-steps=40000
|
| 17 |
+
--train.learning-rate=1e-4
|
| 18 |
+
--train.max-grad-norm=1.0
|
| 19 |
+
--train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 20 |
+
--train.lr_scheduler_kwargs.betas 0.95 0.999
|
| 21 |
+
--train.lr_scheduler_type=cosine
|
| 22 |
+
--data.root_dir=/data/jliu/data
|
| 23 |
+
--data.train-repo-ids=G1WholebodyLocomotionPickBetweenTablesTeleop-v0
|
| 24 |
+
--data.transform.repack.action_chunk_size=30
|
| 25 |
+
--data.transform.repack.pad-action-dim=36
|
| 26 |
+
--data.transform.repack.pad-state-dim=36
|
| 27 |
+
--data.transform.field.stat-path=meta/stats_psi0.json
|
| 28 |
+
--data.transform.field.stat-action-key=action
|
| 29 |
+
--data.transform.field.stat-state-key=states
|
| 30 |
+
--data.transform.field.normalize-state
|
| 31 |
+
--data.transform.field.action-norm-type=bounds
|
| 32 |
+
--data.transform.field.pad-action-dim=36
|
| 33 |
+
--data.transform.field.pad-state-dim=36
|
| 34 |
+
--data.transform.model.img-aug
|
| 35 |
+
--model.chunk-size=30
|
| 36 |
+
--model.n-action-steps=30
|
| 37 |
+
--model.action-dim=36
|
| 38 |
+
--model.state-dim=36
|
| 39 |
+
--model.use-vae
|
| 40 |
+
--model.kl-weight=10.0
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4535d4f0decb1422ba53c4f3d3d639f9362638e31a1cf4bf34a2cccb860c6d57
|
| 3 |
+
size 206658952
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/optimizer.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:71a3663f16b5fc24065782c149cc1257801c83b62702f58b729a1fc345c037f0
|
| 3 |
+
size 413111371
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/random_states_0.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9d45c9d781594007d7b0976207a38ab9be25b60fe7bd7c88e747c0ad60723cea
|
| 3 |
+
size 15473
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/scheduler.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:edd2ef4de5774723b53b76baced1d134ab2a534e3951c7a3ebfbc157468c8f72
|
| 3 |
+
size 1401
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/envs.txt
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
OMP_NUM_THREADS=32
|
| 2 |
+
HF_HOME=/data/cache
|
| 3 |
+
TORCH_HOME=/data/cache
|
| 4 |
+
HF_TOKEN=hf_...TiKa
|
| 5 |
+
HF_LEROBOT_HOME=/data/data/lerobot
|
| 6 |
+
WE_HOME=Not Set
|
| 7 |
+
DATA_HOME=/data/data
|
| 8 |
+
UV_CACHE_DIR=/data/cache
|
| 9 |
+
WANDB_API_KEY=90e...5c06
|
| 10 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
|
| 11 |
+
CUDA_VISIBLE_DEVICES=4,5,6,7
|
| 12 |
+
WORLD_SIZE=4
|
| 13 |
+
LOCAL_WORLD_SIZE=4
|
| 14 |
+
RANK=0
|
| 15 |
+
LOCAL_RANK=0
|
| 16 |
+
MASTER_ADDR=nebula99
|
| 17 |
+
MASTER_PORT=35899
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/run_config.json
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"exp": "g1wholebodylocomotionpickbetweentablesteleop-v0",
|
| 3 |
+
"seed": 2026,
|
| 4 |
+
"auto_tag_run": false,
|
| 5 |
+
"eval": false,
|
| 6 |
+
"debug": false,
|
| 7 |
+
"timestamp": "2604081550",
|
| 8 |
+
"log": {
|
| 9 |
+
"logging_dir": "logs",
|
| 10 |
+
"report_to": "wandb",
|
| 11 |
+
"log_freq": 100
|
| 12 |
+
},
|
| 13 |
+
"wandb": {
|
| 14 |
+
"project": "psi",
|
| 15 |
+
"entity": "jliu530-soochow-university",
|
| 16 |
+
"group": "act-g1",
|
| 17 |
+
"id": "ii0lixdx",
|
| 18 |
+
"name": "g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550",
|
| 19 |
+
"resume": "allow"
|
| 20 |
+
},
|
| 21 |
+
"train": {
|
| 22 |
+
"num_workers": 8,
|
| 23 |
+
"overfit_single_batch": false,
|
| 24 |
+
"name": "act-g1",
|
| 25 |
+
"resume_from_checkpoint": null,
|
| 26 |
+
"skip_resumed_steps": false,
|
| 27 |
+
"hf_token": ".hf_token",
|
| 28 |
+
"lora": false,
|
| 29 |
+
"output_dir": ".runs",
|
| 30 |
+
"gradient_accumulation_steps": 1,
|
| 31 |
+
"mixed_precision": "bf16",
|
| 32 |
+
"max_grad_norm": 1.0,
|
| 33 |
+
"optimizer_foreach": null,
|
| 34 |
+
"train_batch_size": 32,
|
| 35 |
+
"val_batch_size": 16,
|
| 36 |
+
"val_num_batches": 20,
|
| 37 |
+
"checkpointing_steps": 5000,
|
| 38 |
+
"max_checkpoints_to_keep": null,
|
| 39 |
+
"validation_steps": 500,
|
| 40 |
+
"learning_rate": 0.0001,
|
| 41 |
+
"lr_scheduler_type": "cosine",
|
| 42 |
+
"lr_scheduler_kwargs": {
|
| 43 |
+
"betas": [
|
| 44 |
+
0.95,
|
| 45 |
+
0.999
|
| 46 |
+
],
|
| 47 |
+
"weight_decay": 1e-6,
|
| 48 |
+
"eps": 1e-8
|
| 49 |
+
},
|
| 50 |
+
"scheduler_specific_kwargs": {},
|
| 51 |
+
"data_parallel": "ddp",
|
| 52 |
+
"sharding_strategy": "full-shard",
|
| 53 |
+
"deepspeed_config": "/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json",
|
| 54 |
+
"enable_gradient_checkpointing": true,
|
| 55 |
+
"enable_mixed_precision_training": true,
|
| 56 |
+
"reduce_in_full_precision": true,
|
| 57 |
+
"max_training_steps": 40000,
|
| 58 |
+
"num_train_epochs": null,
|
| 59 |
+
"warmup_steps": 1000,
|
| 60 |
+
"warmup_ratio": null
|
| 61 |
+
},
|
| 62 |
+
"data": {
|
| 63 |
+
"transform": {
|
| 64 |
+
"repack": {
|
| 65 |
+
"dataset_name": "simple",
|
| 66 |
+
"num_past_frames": 0,
|
| 67 |
+
"action_chunk_size": 30,
|
| 68 |
+
"pad_action_dim": 36,
|
| 69 |
+
"pad_state_dim": 36
|
| 70 |
+
},
|
| 71 |
+
"model": {
|
| 72 |
+
"resize": {
|
| 73 |
+
"size": [
|
| 74 |
+
256,
|
| 75 |
+
480
|
| 76 |
+
]
|
| 77 |
+
},
|
| 78 |
+
"center_crop": {
|
| 79 |
+
"size": [
|
| 80 |
+
224,
|
| 81 |
+
224
|
| 82 |
+
]
|
| 83 |
+
},
|
| 84 |
+
"color_jitter": {
|
| 85 |
+
"brightness": 0.2,
|
| 86 |
+
"contrast": [
|
| 87 |
+
0.8,
|
| 88 |
+
1.2
|
| 89 |
+
],
|
| 90 |
+
"saturation": [
|
| 91 |
+
0.8,
|
| 92 |
+
1.2
|
| 93 |
+
],
|
| 94 |
+
"hue": 0.05
|
| 95 |
+
},
|
| 96 |
+
"normalize": {
|
| 97 |
+
"mean": [
|
| 98 |
+
0.485,
|
| 99 |
+
0.456,
|
| 100 |
+
0.406
|
| 101 |
+
],
|
| 102 |
+
"std": [
|
| 103 |
+
0.229,
|
| 104 |
+
0.224,
|
| 105 |
+
0.225
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
"img_aug": true
|
| 109 |
+
},
|
| 110 |
+
"field": {
|
| 111 |
+
"stat_path": "meta/stats_psi0.json",
|
| 112 |
+
"action_norm_type": "bounds",
|
| 113 |
+
"stat_action_key": "action",
|
| 114 |
+
"stat_state_key": "states",
|
| 115 |
+
"use_norm_mask": false,
|
| 116 |
+
"action_norm_masks": [
|
| 117 |
+
true,
|
| 118 |
+
true,
|
| 119 |
+
true,
|
| 120 |
+
true,
|
| 121 |
+
true,
|
| 122 |
+
true,
|
| 123 |
+
false
|
| 124 |
+
],
|
| 125 |
+
"action_min": [
|
| 126 |
+
0.0,
|
| 127 |
+
0.0,
|
| 128 |
+
0.0,
|
| 129 |
+
-1.5,
|
| 130 |
+
-1.5,
|
| 131 |
+
-0.6000000238418579,
|
| 132 |
+
-1.5,
|
| 133 |
+
-0.5,
|
| 134 |
+
-0.699999988079071,
|
| 135 |
+
-0.699999988079071,
|
| 136 |
+
-2.216935230032842e-16,
|
| 137 |
+
-2.216935230032842e-16,
|
| 138 |
+
-4.0845591349633594e-18,
|
| 139 |
+
-2.216935230032842e-16,
|
| 140 |
+
-0.4883034825325012,
|
| 141 |
+
0.1900009959936142,
|
| 142 |
+
-0.5470856428146362,
|
| 143 |
+
-0.34318920969963074,
|
| 144 |
+
-0.35952919721603394,
|
| 145 |
+
-0.35302427411079407,
|
| 146 |
+
-0.4469815790653229,
|
| 147 |
+
-0.6371198296546936,
|
| 148 |
+
-0.7683824300765991,
|
| 149 |
+
-1.0653810501098633,
|
| 150 |
+
-0.8479154706001282,
|
| 151 |
+
-1.0297260284423828,
|
| 152 |
+
-0.42936205863952637,
|
| 153 |
+
-0.5147944092750549,
|
| 154 |
+
-0.16820405423641205,
|
| 155 |
+
-0.045328833162784576,
|
| 156 |
+
-0.13282617926597595,
|
| 157 |
+
0.7400000095367432,
|
| 158 |
+
-0.5,
|
| 159 |
+
-0.5,
|
| 160 |
+
-1.0,
|
| 161 |
+
-3.138223648071289
|
| 162 |
+
],
|
| 163 |
+
"action_max": [
|
| 164 |
+
0.5,
|
| 165 |
+
0.699999988079071,
|
| 166 |
+
0.699999988079071,
|
| 167 |
+
0.0,
|
| 168 |
+
0.0,
|
| 169 |
+
0.0,
|
| 170 |
+
0.0,
|
| 171 |
+
6.930528109384597e-19,
|
| 172 |
+
1.108467615016421e-16,
|
| 173 |
+
1.108467615016421e-16,
|
| 174 |
+
1.5,
|
| 175 |
+
1.5,
|
| 176 |
+
1.0,
|
| 177 |
+
1.5,
|
| 178 |
+
0.2557959258556366,
|
| 179 |
+
0.35884979367256165,
|
| 180 |
+
0.5090755820274353,
|
| 181 |
+
0.19132143259048462,
|
| 182 |
+
0.21249642968177795,
|
| 183 |
+
0.3992660641670227,
|
| 184 |
+
0.4283020794391632,
|
| 185 |
+
0.1457289606332779,
|
| 186 |
+
-0.1900009959936142,
|
| 187 |
+
0.6150448322296143,
|
| 188 |
+
0.35468167066574097,
|
| 189 |
+
0.8703295588493347,
|
| 190 |
+
0.7531875371932983,
|
| 191 |
+
0.971237301826477,
|
| 192 |
+
0.13985762000083923,
|
| 193 |
+
0.15686897933483124,
|
| 194 |
+
0.4661160111427307,
|
| 195 |
+
0.7400000095367432,
|
| 196 |
+
0.5,
|
| 197 |
+
0.5,
|
| 198 |
+
1.0,
|
| 199 |
+
3.1414895057678223
|
| 200 |
+
],
|
| 201 |
+
"state_min": [
|
| 202 |
+
-0.02442001923918724,
|
| 203 |
+
-0.0517612099647522,
|
| 204 |
+
-0.0006534014828503132,
|
| 205 |
+
-0.5095356106758118,
|
| 206 |
+
-1.323034405708313,
|
| 207 |
+
-1.3221508264541626,
|
| 208 |
+
-1.3230019807815552,
|
| 209 |
+
-0.5770347714424133,
|
| 210 |
+
-0.4338151812553406,
|
| 211 |
+
-0.6721642017364502,
|
| 212 |
+
-0.0017213862156495452,
|
| 213 |
+
-7.534810038123396e-7,
|
| 214 |
+
-0.001927333534695208,
|
| 215 |
+
-1.075333216249419e-6,
|
| 216 |
+
-0.43650975823402405,
|
| 217 |
+
0.15721464157104492,
|
| 218 |
+
-0.5489339232444763,
|
| 219 |
+
-0.2632291913032532,
|
| 220 |
+
-0.3508843183517456,
|
| 221 |
+
-0.23784859478473663,
|
| 222 |
+
-0.4281824827194214,
|
| 223 |
+
-0.5803383588790894,
|
| 224 |
+
-0.7118590474128723,
|
| 225 |
+
-1.0344431400299072,
|
| 226 |
+
-0.7932196259498596,
|
| 227 |
+
-1.0205217599868774,
|
| 228 |
+
-0.3445618450641632,
|
| 229 |
+
-0.5986371040344238,
|
| 230 |
+
-0.13537253439426422,
|
| 231 |
+
-0.0017330688424408436,
|
| 232 |
+
-0.1421850621700287,
|
| 233 |
+
0.7400000095367432,
|
| 234 |
+
0.0,
|
| 235 |
+
0.0,
|
| 236 |
+
0.0,
|
| 237 |
+
0.0
|
| 238 |
+
],
|
| 239 |
+
"state_max": [
|
| 240 |
+
0.47981399297714233,
|
| 241 |
+
0.6772664189338684,
|
| 242 |
+
0.6746510863304138,
|
| 243 |
+
0.0010172375477850437,
|
| 244 |
+
0.0007091082516126335,
|
| 245 |
+
0.001881288131698966,
|
| 246 |
+
0.0011398319620639086,
|
| 247 |
+
6.141255539660051e-7,
|
| 248 |
+
0.3043450713157654,
|
| 249 |
+
6.343479981296696e-7,
|
| 250 |
+
0.6933000087738037,
|
| 251 |
+
1.4612544775009155,
|
| 252 |
+
1.4651201963424683,
|
| 253 |
+
1.4609057903289795,
|
| 254 |
+
0.2809508740901947,
|
| 255 |
+
0.34028318524360657,
|
| 256 |
+
0.47627460956573486,
|
| 257 |
+
0.26476219296455383,
|
| 258 |
+
0.20825636386871338,
|
| 259 |
+
0.4566418528556824,
|
| 260 |
+
0.42864030599594116,
|
| 261 |
+
0.1656116098165512,
|
| 262 |
+
-0.1549365073442459,
|
| 263 |
+
0.5154499411582947,
|
| 264 |
+
0.4242899715900421,
|
| 265 |
+
0.8548054695129395,
|
| 266 |
+
0.8040095567703247,
|
| 267 |
+
0.9811649322509766,
|
| 268 |
+
0.136736661195755,
|
| 269 |
+
0.195722296833992,
|
| 270 |
+
0.45781663060188293,
|
| 271 |
+
0.7400000095367432,
|
| 272 |
+
0.0,
|
| 273 |
+
0.0,
|
| 274 |
+
0.0,
|
| 275 |
+
0.0
|
| 276 |
+
],
|
| 277 |
+
"normalize_state": true,
|
| 278 |
+
"pad_action_dim": 36,
|
| 279 |
+
"pad_state_dim": 36
|
| 280 |
+
}
|
| 281 |
+
},
|
| 282 |
+
"root_dir": "/data/jliu/data",
|
| 283 |
+
"train_repo_ids": [
|
| 284 |
+
"G1WholebodyLocomotionPickBetweenTablesTeleop-v0"
|
| 285 |
+
],
|
| 286 |
+
"val_repo_ids": [
|
| 287 |
+
"G1WholebodyLocomotionPickBetweenTablesTeleop-v0"
|
| 288 |
+
]
|
| 289 |
+
},
|
| 290 |
+
"model": {
|
| 291 |
+
"n_obs_steps": 1,
|
| 292 |
+
"chunk_size": 30,
|
| 293 |
+
"n_action_steps": 30,
|
| 294 |
+
"action_dim": 36,
|
| 295 |
+
"state_dim": 36,
|
| 296 |
+
"dim_model": 512,
|
| 297 |
+
"n_heads": 8,
|
| 298 |
+
"dim_feedforward": 3200,
|
| 299 |
+
"feedforward_activation": "relu",
|
| 300 |
+
"n_encoder_layers": 4,
|
| 301 |
+
"n_decoder_layers": 1,
|
| 302 |
+
"pre_norm": false,
|
| 303 |
+
"dropout": 0.1,
|
| 304 |
+
"use_vae": true,
|
| 305 |
+
"latent_dim": 32,
|
| 306 |
+
"n_vae_encoder_layers": 4,
|
| 307 |
+
"kl_weight": 10.0,
|
| 308 |
+
"temporal_ensemble_coeff": null
|
| 309 |
+
}
|
| 310 |
+
}
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-08 15:50:54,515 INFO MainThread:641049 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-08 15:50:54,515 INFO MainThread:641049 [wandb_setup.py:_flush():81] Configure stats pid to 641049
|
| 3 |
+
2026-04-08 15:50:54,515 INFO MainThread:641049 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/logs/debug.log
|
| 5 |
+
2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/logs/debug-internal.log
|
| 6 |
+
2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-08 15:50:54,765 INFO MainThread:641049 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-08 15:50:54,774 INFO MainThread:641049 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-08 15:50:54,775 INFO MainThread:641049 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-08 15:50:54,780 INFO MainThread:641049 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-08 15:50:55,555 INFO MainThread:641049 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-08 15:50:55,706 INFO MainThread:641049 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-08 15:50:55,707 INFO MainThread:641049 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-08 15:50:55,707 INFO MainThread:641049 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-08 15:50:55,707 INFO MainThread:641049 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-08 15:50:55,710 INFO MainThread:641049 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-08 15:50:55,712 INFO MainThread:641049 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodylocomotionpickbetweentablesteleop-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2604081550', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [0.0, 0.0, 0.0, -1.5, -1.5, -0.6000000238418579, -1.5, -0.5, -0.699999988079071, -0.699999988079071, -2.216935230032842e-16, -2.216935230032842e-16, -4.0845591349633594e-18, -2.216935230032842e-16, -0.4883034825325012, 0.1900009959936142, -0.5470856428146362, -0.34318920969963074, -0.35952919721603394, -0.35302427411079407, -0.4469815790653229, -0.6371198296546936, -0.7683824300765991, -1.0653810501098633, -0.8479154706001282, -1.0297260284423828, -0.42936205863952637, -0.5147944092750549, -0.16820405423641205, -0.045328833162784576, -0.13282617926597595, 0.7400000095367432, -0.5, -0.5, -1.0, -3.138223648071289], 'action_max': [0.5, 0.699999988079071, 0.699999988079071, 0.0, 0.0, 0.0, 0.0, 6.930528109384597e-19, 1.108467615016421e-16, 1.108467615016421e-16, 1.5, 1.5, 1.0, 1.5, 0.2557959258556366, 0.35884979367256165, 0.5090755820274353, 0.19132143259048462, 0.21249642968177795, 0.3992660641670227, 0.4283020794391632, 0.1457289606332779, -0.1900009959936142, 0.6150448322296143, 0.35468167066574097, 0.8703295588493347, 0.7531875371932983, 0.971237301826477, 0.13985762000083923, 0.15686897933483124, 0.4661160111427307, 0.7400000095367432, 0.5, 0.5, 1.0, 3.1414895057678223], 'state_min': [-0.02442001923918724, -0.0517612099647522, -0.0006534014828503132, -0.5095356106758118, -1.323034405708313, -1.3221508264541626, -1.3230019807815552, -0.5770347714424133, -0.4338151812553406, -0.6721642017364502, -0.0017213862156495452, -7.534810038123396e-07, -0.001927333534695208, -1.075333216249419e-06, -0.43650975823402405, 0.15721464157104492, -0.5489339232444763, -0.2632291913032532, -0.3508843183517456, -0.23784859478473663, -0.4281824827194214, -0.5803383588790894, -0.7118590474128723, -1.0344431400299072, -0.7932196259498596, -1.0205217599868774, -0.3445618450641632, -0.5986371040344238, -0.13537253439426422, -0.0017330688424408436, -0.1421850621700287, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.47981399297714233, 0.6772664189338684, 0.6746510863304138, 0.0010172375477850437, 0.0007091082516126335, 0.001881288131698966, 0.0011398319620639086, 6.141255539660051e-07, 0.3043450713157654, 6.343479981296696e-07, 0.6933000087738037, 1.4612544775009155, 1.4651201963424683, 1.4609057903289795, 0.2809508740901947, 0.34028318524360657, 0.47627460956573486, 0.26476219296455383, 0.20825636386871338, 0.4566418528556824, 0.42864030599594116, 0.1656116098165512, -0.1549365073442459, 0.5154499411582947, 0.4242899715900421, 0.8548054695129395, 0.8040095567703247, 0.9811649322509766, 0.136736661195755, 0.195722296833992, 0.45781663060188293, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyLocomotionPickBetweenTablesTeleop-v0'], 'val_repo_ids': ['G1WholebodyLocomotionPickBetweenTablesTeleop-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '4,5,6,7', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '35899'}}
|
| 21 |
+
2026-04-08 19:34:32,424 INFO MainThread:641049 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/ii0lixdx
|
| 22 |
+
2026-04-08 19:34:32,427 INFO MainThread:641049 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
|
| 23 |
+
2026-04-08 19:34:32,427 INFO MainThread:641049 [wandb_run.py:_restore():2472] restore
|
| 24 |
+
2026-04-08 19:34:32,427 INFO MainThread:641049 [wandb_run.py:_restore():2478] restore done
|
| 25 |
+
2026-04-08 19:34:33,334 INFO MainThread:641049 [wandb_run.py:_footer_sync_info():3868] logging synced files
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/config.yaml
ADDED
|
@@ -0,0 +1,448 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.1
|
| 4 |
+
e:
|
| 5 |
+
ufhcsemgf4luglllwa36q76bqs2rc5qh:
|
| 6 |
+
args:
|
| 7 |
+
- simple_act_config
|
| 8 |
+
- --seed=2026
|
| 9 |
+
- --exp=g1wholebodylocomotionpickbetweentablesteleop-v0
|
| 10 |
+
- --train.name=act-g1
|
| 11 |
+
- --log.report-to=wandb
|
| 12 |
+
- --train.data_parallel=ddp
|
| 13 |
+
- --train.mixed_precision=bf16
|
| 14 |
+
- --train.train-batch-size=32
|
| 15 |
+
- --train.warmup-steps=1000
|
| 16 |
+
- --train.warmup-ratio=None
|
| 17 |
+
- --train.checkpointing-steps=5000
|
| 18 |
+
- --train.validation_steps=500
|
| 19 |
+
- --train.val_num_batches=20
|
| 20 |
+
- --train.gradient_accumulation_steps=1
|
| 21 |
+
- --train.max-training-steps=40000
|
| 22 |
+
- --train.learning-rate=1e-4
|
| 23 |
+
- --train.max-grad-norm=1.0
|
| 24 |
+
- --train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 25 |
+
- --train.lr_scheduler_kwargs.betas
|
| 26 |
+
- "0.95"
|
| 27 |
+
- "0.999"
|
| 28 |
+
- --train.lr_scheduler_type=cosine
|
| 29 |
+
- --data.root_dir=/data/jliu/data
|
| 30 |
+
- --data.train-repo-ids=G1WholebodyLocomotionPickBetweenTablesTeleop-v0
|
| 31 |
+
- --data.transform.repack.action_chunk_size=30
|
| 32 |
+
- --data.transform.repack.pad-action-dim=36
|
| 33 |
+
- --data.transform.repack.pad-state-dim=36
|
| 34 |
+
- --data.transform.field.stat-path=meta/stats_psi0.json
|
| 35 |
+
- --data.transform.field.stat-action-key=action
|
| 36 |
+
- --data.transform.field.stat-state-key=states
|
| 37 |
+
- --data.transform.field.normalize-state
|
| 38 |
+
- --data.transform.field.action-norm-type=bounds
|
| 39 |
+
- --data.transform.field.pad-action-dim=36
|
| 40 |
+
- --data.transform.field.pad-state-dim=36
|
| 41 |
+
- --data.transform.model.img-aug
|
| 42 |
+
- --model.chunk-size=30
|
| 43 |
+
- --model.n-action-steps=30
|
| 44 |
+
- --model.action-dim=36
|
| 45 |
+
- --model.state-dim=36
|
| 46 |
+
- --model.use-vae
|
| 47 |
+
- --model.kl-weight=10.0
|
| 48 |
+
codePath: scripts/train.py
|
| 49 |
+
codePathLocal: scripts/train.py
|
| 50 |
+
cpu_count: 128
|
| 51 |
+
cpu_count_logical: 128
|
| 52 |
+
cudaVersion: "12.9"
|
| 53 |
+
disk:
|
| 54 |
+
/:
|
| 55 |
+
total: "105089261568"
|
| 56 |
+
used: "92984258560"
|
| 57 |
+
email: jliu530@163.com
|
| 58 |
+
executable: /data/jliu/psi/.venv-psi/bin/python3
|
| 59 |
+
git:
|
| 60 |
+
commit: 04614628ecb677f5f278e2d31c1103aed8127f26
|
| 61 |
+
remote: https://github.com/songlin/psi.git
|
| 62 |
+
gpu: NVIDIA A100-SXM4-80GB
|
| 63 |
+
gpu_count: 8
|
| 64 |
+
gpu_nvidia:
|
| 65 |
+
- architecture: Ampere
|
| 66 |
+
cudaCores: 6912
|
| 67 |
+
memoryTotal: "85899345920"
|
| 68 |
+
name: NVIDIA A100-SXM4-80GB
|
| 69 |
+
uuid: GPU-02b51758-2aec-8c71-c751-92567fdb15ad
|
| 70 |
+
- architecture: Ampere
|
| 71 |
+
cudaCores: 6912
|
| 72 |
+
memoryTotal: "85899345920"
|
| 73 |
+
name: NVIDIA A100-SXM4-80GB
|
| 74 |
+
uuid: GPU-be645bb4-1ef0-15da-fc6b-53c5378650df
|
| 75 |
+
- architecture: Ampere
|
| 76 |
+
cudaCores: 6912
|
| 77 |
+
memoryTotal: "85899345920"
|
| 78 |
+
name: NVIDIA A100-SXM4-80GB
|
| 79 |
+
uuid: GPU-44868ccd-5809-61e2-f8cc-1dff8efc7eb1
|
| 80 |
+
- architecture: Ampere
|
| 81 |
+
cudaCores: 6912
|
| 82 |
+
memoryTotal: "85899345920"
|
| 83 |
+
name: NVIDIA A100-SXM4-80GB
|
| 84 |
+
uuid: GPU-a12b8de3-fb12-2901-c3a2-1e0e2b050b7b
|
| 85 |
+
- architecture: Ampere
|
| 86 |
+
cudaCores: 6912
|
| 87 |
+
memoryTotal: "85899345920"
|
| 88 |
+
name: NVIDIA A100-SXM4-80GB
|
| 89 |
+
uuid: GPU-57787183-4904-b2c9-cf52-1c1af976b365
|
| 90 |
+
- architecture: Ampere
|
| 91 |
+
cudaCores: 6912
|
| 92 |
+
memoryTotal: "85899345920"
|
| 93 |
+
name: NVIDIA A100-SXM4-80GB
|
| 94 |
+
uuid: GPU-e82d0fd0-9dcb-3d7c-9451-c4770ad6ddc3
|
| 95 |
+
- architecture: Ampere
|
| 96 |
+
cudaCores: 6912
|
| 97 |
+
memoryTotal: "85899345920"
|
| 98 |
+
name: NVIDIA A100-SXM4-80GB
|
| 99 |
+
uuid: GPU-9a825fa3-60d0-3acf-6c79-67e9230ae24c
|
| 100 |
+
- architecture: Ampere
|
| 101 |
+
cudaCores: 6912
|
| 102 |
+
memoryTotal: "85899345920"
|
| 103 |
+
name: NVIDIA A100-SXM4-80GB
|
| 104 |
+
uuid: GPU-6fe02e5e-b886-9c02-e433-6237f3897dd8
|
| 105 |
+
host: nebula99
|
| 106 |
+
memory:
|
| 107 |
+
total: "1623177445376"
|
| 108 |
+
os: Linux-6.8.0-106-generic-x86_64-with-glibc2.39
|
| 109 |
+
program: /data/jliu/psi/scripts/train.py
|
| 110 |
+
python: CPython 3.10.20
|
| 111 |
+
root: /data/jliu/psi/.runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550
|
| 112 |
+
startedAt: "2026-04-08T15:50:54.514093Z"
|
| 113 |
+
writerId: ufhcsemgf4luglllwa36q76bqs2rc5qh
|
| 114 |
+
m: []
|
| 115 |
+
python_version: 3.10.20
|
| 116 |
+
t:
|
| 117 |
+
"1":
|
| 118 |
+
- 1
|
| 119 |
+
- 11
|
| 120 |
+
- 41
|
| 121 |
+
- 49
|
| 122 |
+
- 71
|
| 123 |
+
"2":
|
| 124 |
+
- 1
|
| 125 |
+
- 11
|
| 126 |
+
- 41
|
| 127 |
+
- 49
|
| 128 |
+
- 51
|
| 129 |
+
- 71
|
| 130 |
+
- 83
|
| 131 |
+
"3":
|
| 132 |
+
- 2
|
| 133 |
+
- 13
|
| 134 |
+
- 61
|
| 135 |
+
"4": 3.10.20
|
| 136 |
+
"5": 0.25.1
|
| 137 |
+
"6": 4.57.0
|
| 138 |
+
"12": 0.25.1
|
| 139 |
+
"13": linux-x86_64
|
| 140 |
+
auto_tag_run:
|
| 141 |
+
value: false
|
| 142 |
+
data:
|
| 143 |
+
value:
|
| 144 |
+
root_dir: /data/jliu/data
|
| 145 |
+
train_repo_ids:
|
| 146 |
+
- G1WholebodyLocomotionPickBetweenTablesTeleop-v0
|
| 147 |
+
transform:
|
| 148 |
+
field:
|
| 149 |
+
action_max:
|
| 150 |
+
- 0.5
|
| 151 |
+
- 0.699999988079071
|
| 152 |
+
- 0.699999988079071
|
| 153 |
+
- 0
|
| 154 |
+
- 0
|
| 155 |
+
- 0
|
| 156 |
+
- 0
|
| 157 |
+
- 6.930528109384597e-19
|
| 158 |
+
- 1.108467615016421e-16
|
| 159 |
+
- 1.108467615016421e-16
|
| 160 |
+
- 1.5
|
| 161 |
+
- 1.5
|
| 162 |
+
- 1
|
| 163 |
+
- 1.5
|
| 164 |
+
- 0.2557959258556366
|
| 165 |
+
- 0.35884979367256165
|
| 166 |
+
- 0.5090755820274353
|
| 167 |
+
- 0.19132143259048462
|
| 168 |
+
- 0.21249642968177795
|
| 169 |
+
- 0.3992660641670227
|
| 170 |
+
- 0.4283020794391632
|
| 171 |
+
- 0.1457289606332779
|
| 172 |
+
- -0.1900009959936142
|
| 173 |
+
- 0.6150448322296143
|
| 174 |
+
- 0.35468167066574097
|
| 175 |
+
- 0.8703295588493347
|
| 176 |
+
- 0.7531875371932983
|
| 177 |
+
- 0.971237301826477
|
| 178 |
+
- 0.13985762000083923
|
| 179 |
+
- 0.15686897933483124
|
| 180 |
+
- 0.4661160111427307
|
| 181 |
+
- 0.7400000095367432
|
| 182 |
+
- 0.5
|
| 183 |
+
- 0.5
|
| 184 |
+
- 1
|
| 185 |
+
- 3.1414895057678223
|
| 186 |
+
action_min:
|
| 187 |
+
- 0
|
| 188 |
+
- 0
|
| 189 |
+
- 0
|
| 190 |
+
- -1.5
|
| 191 |
+
- -1.5
|
| 192 |
+
- -0.6000000238418579
|
| 193 |
+
- -1.5
|
| 194 |
+
- -0.5
|
| 195 |
+
- -0.699999988079071
|
| 196 |
+
- -0.699999988079071
|
| 197 |
+
- -2.216935230032842e-16
|
| 198 |
+
- -2.216935230032842e-16
|
| 199 |
+
- -4.0845591349633594e-18
|
| 200 |
+
- -2.216935230032842e-16
|
| 201 |
+
- -0.4883034825325012
|
| 202 |
+
- 0.1900009959936142
|
| 203 |
+
- -0.5470856428146362
|
| 204 |
+
- -0.34318920969963074
|
| 205 |
+
- -0.35952919721603394
|
| 206 |
+
- -0.35302427411079407
|
| 207 |
+
- -0.4469815790653229
|
| 208 |
+
- -0.6371198296546936
|
| 209 |
+
- -0.7683824300765991
|
| 210 |
+
- -1.0653810501098633
|
| 211 |
+
- -0.8479154706001282
|
| 212 |
+
- -1.0297260284423828
|
| 213 |
+
- -0.42936205863952637
|
| 214 |
+
- -0.5147944092750549
|
| 215 |
+
- -0.16820405423641205
|
| 216 |
+
- -0.045328833162784576
|
| 217 |
+
- -0.13282617926597595
|
| 218 |
+
- 0.7400000095367432
|
| 219 |
+
- -0.5
|
| 220 |
+
- -0.5
|
| 221 |
+
- -1
|
| 222 |
+
- -3.138223648071289
|
| 223 |
+
action_norm_masks:
|
| 224 |
+
- true
|
| 225 |
+
- true
|
| 226 |
+
- true
|
| 227 |
+
- true
|
| 228 |
+
- true
|
| 229 |
+
- true
|
| 230 |
+
- false
|
| 231 |
+
action_norm_type: bounds
|
| 232 |
+
normalize_state: true
|
| 233 |
+
pad_action_dim: 36
|
| 234 |
+
pad_state_dim: 36
|
| 235 |
+
stat_action_key: action
|
| 236 |
+
stat_path: meta/stats_psi0.json
|
| 237 |
+
stat_state_key: states
|
| 238 |
+
state_max:
|
| 239 |
+
- 0.47981399297714233
|
| 240 |
+
- 0.6772664189338684
|
| 241 |
+
- 0.6746510863304138
|
| 242 |
+
- 0.0010172375477850437
|
| 243 |
+
- 0.0007091082516126335
|
| 244 |
+
- 0.001881288131698966
|
| 245 |
+
- 0.0011398319620639086
|
| 246 |
+
- 6.141255539660051e-07
|
| 247 |
+
- 0.3043450713157654
|
| 248 |
+
- 6.343479981296696e-07
|
| 249 |
+
- 0.6933000087738037
|
| 250 |
+
- 1.4612544775009155
|
| 251 |
+
- 1.4651201963424683
|
| 252 |
+
- 1.4609057903289795
|
| 253 |
+
- 0.2809508740901947
|
| 254 |
+
- 0.34028318524360657
|
| 255 |
+
- 0.47627460956573486
|
| 256 |
+
- 0.26476219296455383
|
| 257 |
+
- 0.20825636386871338
|
| 258 |
+
- 0.4566418528556824
|
| 259 |
+
- 0.42864030599594116
|
| 260 |
+
- 0.1656116098165512
|
| 261 |
+
- -0.1549365073442459
|
| 262 |
+
- 0.5154499411582947
|
| 263 |
+
- 0.4242899715900421
|
| 264 |
+
- 0.8548054695129395
|
| 265 |
+
- 0.8040095567703247
|
| 266 |
+
- 0.9811649322509766
|
| 267 |
+
- 0.136736661195755
|
| 268 |
+
- 0.195722296833992
|
| 269 |
+
- 0.45781663060188293
|
| 270 |
+
- 0.7400000095367432
|
| 271 |
+
- 0
|
| 272 |
+
- 0
|
| 273 |
+
- 0
|
| 274 |
+
- 0
|
| 275 |
+
state_min:
|
| 276 |
+
- -0.02442001923918724
|
| 277 |
+
- -0.0517612099647522
|
| 278 |
+
- -0.0006534014828503132
|
| 279 |
+
- -0.5095356106758118
|
| 280 |
+
- -1.323034405708313
|
| 281 |
+
- -1.3221508264541626
|
| 282 |
+
- -1.3230019807815552
|
| 283 |
+
- -0.5770347714424133
|
| 284 |
+
- -0.4338151812553406
|
| 285 |
+
- -0.6721642017364502
|
| 286 |
+
- -0.0017213862156495452
|
| 287 |
+
- -7.534810038123396e-07
|
| 288 |
+
- -0.001927333534695208
|
| 289 |
+
- -1.075333216249419e-06
|
| 290 |
+
- -0.43650975823402405
|
| 291 |
+
- 0.15721464157104492
|
| 292 |
+
- -0.5489339232444763
|
| 293 |
+
- -0.2632291913032532
|
| 294 |
+
- -0.3508843183517456
|
| 295 |
+
- -0.23784859478473663
|
| 296 |
+
- -0.4281824827194214
|
| 297 |
+
- -0.5803383588790894
|
| 298 |
+
- -0.7118590474128723
|
| 299 |
+
- -1.0344431400299072
|
| 300 |
+
- -0.7932196259498596
|
| 301 |
+
- -1.0205217599868774
|
| 302 |
+
- -0.3445618450641632
|
| 303 |
+
- -0.5986371040344238
|
| 304 |
+
- -0.13537253439426422
|
| 305 |
+
- -0.0017330688424408436
|
| 306 |
+
- -0.1421850621700287
|
| 307 |
+
- 0.7400000095367432
|
| 308 |
+
- 0
|
| 309 |
+
- 0
|
| 310 |
+
- 0
|
| 311 |
+
- 0
|
| 312 |
+
use_norm_mask: false
|
| 313 |
+
model:
|
| 314 |
+
center_crop:
|
| 315 |
+
size:
|
| 316 |
+
- 224
|
| 317 |
+
- 224
|
| 318 |
+
color_jitter:
|
| 319 |
+
brightness: 0.2
|
| 320 |
+
contrast:
|
| 321 |
+
- 0.8
|
| 322 |
+
- 1.2
|
| 323 |
+
hue: 0.05
|
| 324 |
+
saturation:
|
| 325 |
+
- 0.8
|
| 326 |
+
- 1.2
|
| 327 |
+
img_aug: true
|
| 328 |
+
normalize:
|
| 329 |
+
mean:
|
| 330 |
+
- 0.485
|
| 331 |
+
- 0.456
|
| 332 |
+
- 0.406
|
| 333 |
+
std:
|
| 334 |
+
- 0.229
|
| 335 |
+
- 0.224
|
| 336 |
+
- 0.225
|
| 337 |
+
resize:
|
| 338 |
+
size:
|
| 339 |
+
- 256
|
| 340 |
+
- 480
|
| 341 |
+
repack:
|
| 342 |
+
action_chunk_size: 30
|
| 343 |
+
dataset_name: simple
|
| 344 |
+
num_past_frames: 0
|
| 345 |
+
pad_action_dim: 36
|
| 346 |
+
pad_state_dim: 36
|
| 347 |
+
val_repo_ids:
|
| 348 |
+
- G1WholebodyLocomotionPickBetweenTablesTeleop-v0
|
| 349 |
+
debug:
|
| 350 |
+
value: false
|
| 351 |
+
environment_variables:
|
| 352 |
+
value:
|
| 353 |
+
CUDA_VISIBLE_DEVICES: 4,5,6,7
|
| 354 |
+
DATA_HOME: /data/data
|
| 355 |
+
HF_HOME: /data/cache
|
| 356 |
+
HF_LEROBOT_HOME: /data/data/lerobot
|
| 357 |
+
HF_TOKEN: hf_...TiKa
|
| 358 |
+
LOCAL_RANK: "0"
|
| 359 |
+
LOCAL_WORLD_SIZE: "4"
|
| 360 |
+
MASTER_ADDR: nebula99
|
| 361 |
+
MASTER_PORT: "35899"
|
| 362 |
+
OMP_NUM_THREADS: "32"
|
| 363 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
|
| 364 |
+
RANK: "0"
|
| 365 |
+
TORCH_HOME: /data/cache
|
| 366 |
+
UV_CACHE_DIR: /data/cache
|
| 367 |
+
WANDB_API_KEY: 90e...5c06
|
| 368 |
+
WE_HOME: Not Set
|
| 369 |
+
WORLD_SIZE: "4"
|
| 370 |
+
eval:
|
| 371 |
+
value: false
|
| 372 |
+
exp:
|
| 373 |
+
value: g1wholebodylocomotionpickbetweentablesteleop-v0
|
| 374 |
+
log:
|
| 375 |
+
value:
|
| 376 |
+
log_freq: 100
|
| 377 |
+
logging_dir: logs
|
| 378 |
+
report_to: wandb
|
| 379 |
+
model:
|
| 380 |
+
value:
|
| 381 |
+
action_dim: 36
|
| 382 |
+
chunk_size: 30
|
| 383 |
+
dim_feedforward: 3200
|
| 384 |
+
dim_model: 512
|
| 385 |
+
dropout: 0.1
|
| 386 |
+
feedforward_activation: relu
|
| 387 |
+
kl_weight: 10
|
| 388 |
+
latent_dim: 32
|
| 389 |
+
n_action_steps: 30
|
| 390 |
+
n_decoder_layers: 1
|
| 391 |
+
n_encoder_layers: 4
|
| 392 |
+
n_heads: 8
|
| 393 |
+
n_obs_steps: 1
|
| 394 |
+
n_vae_encoder_layers: 4
|
| 395 |
+
pre_norm: false
|
| 396 |
+
state_dim: 36
|
| 397 |
+
temporal_ensemble_coeff: null
|
| 398 |
+
use_vae: true
|
| 399 |
+
seed:
|
| 400 |
+
value: 2026
|
| 401 |
+
timestamp:
|
| 402 |
+
value: "2604081550"
|
| 403 |
+
train:
|
| 404 |
+
value:
|
| 405 |
+
checkpointing_steps: 5000
|
| 406 |
+
data_parallel: ddp
|
| 407 |
+
deepspeed_config: /data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json
|
| 408 |
+
enable_gradient_checkpointing: true
|
| 409 |
+
enable_mixed_precision_training: true
|
| 410 |
+
gradient_accumulation_steps: 1
|
| 411 |
+
hf_token: .hf_token
|
| 412 |
+
learning_rate: 0.0001
|
| 413 |
+
lora: false
|
| 414 |
+
lr_scheduler_kwargs:
|
| 415 |
+
betas:
|
| 416 |
+
- 0.95
|
| 417 |
+
- 0.999
|
| 418 |
+
eps: 1e-08
|
| 419 |
+
weight_decay: 1e-06
|
| 420 |
+
lr_scheduler_type: cosine
|
| 421 |
+
max_checkpoints_to_keep: null
|
| 422 |
+
max_grad_norm: 1
|
| 423 |
+
max_training_steps: 40000
|
| 424 |
+
mixed_precision: bf16
|
| 425 |
+
name: act-g1
|
| 426 |
+
num_train_epochs: null
|
| 427 |
+
num_workers: 8
|
| 428 |
+
optimizer_foreach: null
|
| 429 |
+
output_dir: .runs
|
| 430 |
+
overfit_single_batch: false
|
| 431 |
+
reduce_in_full_precision: true
|
| 432 |
+
resume_from_checkpoint: null
|
| 433 |
+
sharding_strategy: full-shard
|
| 434 |
+
skip_resumed_steps: false
|
| 435 |
+
train_batch_size: 32
|
| 436 |
+
val_batch_size: 16
|
| 437 |
+
val_num_batches: 20
|
| 438 |
+
validation_steps: 500
|
| 439 |
+
warmup_ratio: null
|
| 440 |
+
warmup_steps: 1000
|
| 441 |
+
wandb:
|
| 442 |
+
value:
|
| 443 |
+
entity: jliu530-soochow-university
|
| 444 |
+
group: null
|
| 445 |
+
id: null
|
| 446 |
+
name: null
|
| 447 |
+
project: psi
|
| 448 |
+
resume: allow
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/output.log
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2;36m[15:50:55 04/08][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Saved configuration to ]8;id=805427;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=514750;file:///data/jliu/psi/scripts/train.py#128\[2m128[0m]8;;\
|
| 2 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m [2m [0m
|
| 3 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Training configurations: ]8;id=2657;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=644039;file:///data/jliu/psi/scripts/train.py#181\[2m181[0m]8;;\
|
| 4 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> training task: [32m'act-g1'[0m ]8;id=471497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=12046;file:///data/jliu/psi/scripts/train.py#182\[2m182[0m]8;;\
|
| 5 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> run name: ]8;id=329497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=220436;file:///data/jliu/psi/scripts/train.py#183\[2m183[0m]8;;\
|
| 6 |
+
[2;36m [0m g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m [2m [0m
|
| 7 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> seed: [1;36m2026[0m ]8;id=844246;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=394625;file:///data/jliu/psi/scripts/train.py#184\[2m184[0m]8;;\
|
| 8 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=94020;file:///data/jliu/psi/scripts/train.py#185\[2m185[0m]8;;\
|
| 9 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> warmup steps: [1;36m1000[0m ]8;id=147732;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=919050;file:///data/jliu/psi/scripts/train.py#186\[2m186[0m]8;;\
|
| 10 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> validation steps: [1;36m500[0m ]8;id=25941;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=888976;file:///data/jliu/psi/scripts/train.py#187\[2m187[0m]8;;\
|
| 11 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> checkpoint steps: [1;36m5000[0m ]8;id=95089;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=418144;file:///data/jliu/psi/scripts/train.py#188\[2m188[0m]8;;\
|
| 12 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> max gradient norm: [1;36m1.0[0m ]8;id=122104;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=446109;file:///data/jliu/psi/scripts/train.py#189\[2m189[0m]8;;\
|
| 13 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> PyTorch version [1;36m2.7[0m.[1;36m0[0m available. ]8;id=514883;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py\[2mconfig.py[0m]8;;\[2m:[0m]8;id=417488;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py#54\[2m54[0m]8;;\
|
| 14 |
+
Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████| 99/99 [00:00<00:00, 178442.67it/s]
|
| 15 |
+
Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████| 99/99 [00:00<00:00, 200888.29it/s]
|
| 16 |
+
[2;36m[15:50:58 04/08][0m[2;36m [0m[34mINFO [0m | >> |=> Num training samples: ]8;id=591396;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=971225;file:///data/jliu/psi/scripts/train.py#192\[2m192[0m]8;;\
|
| 17 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Training dataset size: [1;36m62[0m,[1;36m764[0m ]8;id=603472;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=235431;file:///data/jliu/psi/scripts/train.py#193\[2m193[0m]8;;\
|
| 18 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Val dataset size: [1;36m62[0m,[1;36m764[0m ]8;id=548632;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=929903;file:///data/jliu/psi/scripts/train.py#195\[2m195[0m]8;;\
|
| 19 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Initialize optimizers and schedulers[33m...[0m ]8;id=492873;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=149330;file:///data/jliu/psi/scripts/train.py#198\[2m198[0m]8;;\
|
| 20 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m ***** Running training ***** ]8;id=127940;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=657111;file:///data/jliu/psi/scripts/train.py#202\[2m202[0m]8;;\
|
| 21 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num training examples = [1;36m62764[0m ]8;id=974614;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=572136;file:///data/jliu/psi/scripts/train.py#203\[2m203[0m]8;;\
|
| 22 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Max training Epochs = [1;36m82[0m ]8;id=574775;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=838653;file:///data/jliu/psi/scripts/train.py#204\[2m204[0m]8;;\
|
| 23 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Total optimization steps = [1;36m40000[0m ]8;id=927413;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=421473;file:///data/jliu/psi/scripts/train.py#205\[2m205[0m]8;;\
|
| 24 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num steps Per Epoch = [1;36m491[0m ]8;id=9727;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=34717;file:///data/jliu/psi/scripts/train.py#206\[2m206[0m]8;;\
|
| 25 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Effective training epochs = [1;36m325.87[0m ]8;id=842580;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=548288;file:///data/jliu/psi/scripts/train.py#207\[2m207[0m]8;;\
|
| 26 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Global train batch size [1m([0mw. parallel, distributed & accumulation[1m)[0m = [1;36m128[0m ]8;id=272745;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=726293;file:///data/jliu/psi/scripts/train.py#208\[2m208[0m]8;;\
|
| 27 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Device train batch size = [1;36m32[0m ]8;id=521307;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=847206;file:///data/jliu/psi/scripts/train.py#209\[2m209[0m]8;;\
|
| 28 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Gradient Accumulation steps = [1;36m1[0m ]8;id=871789;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=304159;file:///data/jliu/psi/scripts/train.py#210\[2m210[0m]8;;\
|
| 29 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num processes [1m([0mGPUs[1m)[0m = [1;36m4[0m ]8;id=459749;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=722818;file:///data/jliu/psi/scripts/train.py#211\[2m211[0m]8;;\
|
| 30 |
+
[2;36m[15:50:59 04/08][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Accelerator runs in: ]8;id=966756;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=191940;file:///data/jliu/psi/scripts/train.py#218\[2m218[0m]8;;\
|
| 31 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m [2m [0m
|
| 32 |
+
Traing steps: 12%|███████▋ | 4999/40000 [27:49<2:56:12, 3.31it/s, loss=0.0669, lr=9.7e-05]
|
| 33 |
+
[2026-04-08 15:51:05,959] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 34 |
+
[2;36m[15:51:06 04/08][0m[2;36m [0m[34mINFO [0m | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=680052;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=279409;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 35 |
+
[2;36m [0m [35m/tmp/tmphf7av35s/[0m[95mtest.c[0m -o [35m/tmp/tmphf7av35s/[0m[95mtest.o[0m [2m [0m
|
| 36 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread [35m/tmp/tmphf7av35s/[0m[95mtest.o[0m -laio -o [35m/tmp/tmphf7av35s/[0m[95ma.out[0m ]8;id=204038;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=783864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 37 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall ]8;id=960999;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=14714;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 38 |
+
[2;36m [0m -O3 -fPIC -fPIC -c [35m/tmp/tmprste3cr8/[0m[95mtest.c[0m -o [35m/tmp/tmprste3cr8/[0m[95mtest.o[0m [2m [0m
|
| 39 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread [35m/tmp/tmprste3cr8/[0m[95mtest.o[0m -L/usr/local/cuda -L/usr/local/cuda/lib64 -lcufile -o ]8;id=573141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=977459;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 40 |
+
[2;36m [0m [35m/tmp/tmprste3cr8/[0m[95ma.out[0m [2m [0m
|
| 41 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall ]8;id=208448;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=259876;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 42 |
+
[2;36m [0m -O3 -fPIC -fPIC -c [35m/tmp/tmp2xda_ky0/[0m[95mtest.c[0m -o [35m/tmp/tmp2xda_ky0/[0m[95mtest.o[0m [2m [0m
|
| 43 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> cc -pthread [35m/tmp/tmp2xda_ky0/[0m[95mtest.o[0m -laio -o [35m/tmp/tmp2xda_ky0/[0m[95ma.out[0m ]8;id=572992;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\[2mspawn.py[0m]8;;\[2m:[0m]8;id=281580;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\[2m77[0m]8;;\
|
| 44 |
+
[2026-04-08 15:51:07,435] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
|
| 45 |
+
|
| 46 |
+
[2;36m[16:18:49 04/08][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=328668;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=696864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 47 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_5000 [2m [0m
|
| 48 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=515887;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=723262;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 49 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_5000/mode [2m [0m
|
| 50 |
+
[2;36m [0m l.safetensors [2m [0m
|
| 51 |
+
[2;36m[16:18:50 04/08][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=495959;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=616494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 52 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_5000/opti [2m [0m
|
| 53 |
+
[2;36m [0m mizer.bin [2m [0m
|
| 54 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=304800;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=88534;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 55 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_5000/sche [2m [0m
|
| 56 |
+
[2;36m [0m duler.bin [2m [0m
|
| 57 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=821494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=820417;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 58 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_5000/samp [2m [0m
|
| 59 |
+
[2;36m [0m ler.bin [2m [0m
|
| 60 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=432054;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=409371;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 61 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_5000/samp [2m [0m
|
| 62 |
+
[2;36m [0m ler_1.bin [2m [0m
|
| 63 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=394271;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=14818;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 64 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_5000/rand [2m [0m
|
| 65 |
+
[2;36m [0m om_states_0.pkl [2m [0m
|
| 66 |
+
Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_5000
|
| 67 |
+
[2;36m[16:45:48 04/08][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=53870;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=874302;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 68 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_10000 [2m [0m
|
| 69 |
+
[2;36m[16:45:49 04/08][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=806914;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=990978;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 70 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_10000/mod [2m [0m
|
| 71 |
+
[2;36m [0m el.safetensors [2m [0m
|
| 72 |
+
[2;36m[16:45:50 04/08][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=343681;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=715961;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 73 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_10000/opt [2m [0m
|
| 74 |
+
[2;36m [0m imizer.bin [2m [0m
|
| 75 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=127268;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=521739;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 76 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_10000/sch [2m [0m
|
| 77 |
+
[2;36m [0m eduler.bin [2m [0m
|
| 78 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=844039;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=179027;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 79 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_10000/sam [2m [0m
|
| 80 |
+
[2;36m [0m pler.bin [2m [0m
|
| 81 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=454670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=653622;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 82 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_10000/sam [2m [0m
|
| 83 |
+
[2;36m [0m pler_1.bin [2m [0m
|
| 84 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=711457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=745414;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 85 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_10000/ran [2m [0m
|
| 86 |
+
[2;36m [0m dom_states_0.pkl [2m [0m
|
| 87 |
+
Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_10000
|
| 88 |
+
[2;36m[17:12:37 04/08][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=204990;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=678670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 89 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_15000 [2m [0m
|
| 90 |
+
[2;36m[17:12:38 04/08][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=571229;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=976178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 91 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_15000/mod [2m [0m
|
| 92 |
+
[2;36m [0m el.safetensors [2m [0m
|
| 93 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=791370;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=671982;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 94 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_15000/opt [2m [0m
|
| 95 |
+
[2;36m [0m imizer.bin [2m [0m
|
| 96 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=920457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=21464;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 97 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_15000/sch [2m [0m
|
| 98 |
+
[2;36m [0m eduler.bin [2m [0m
|
| 99 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=823560;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=925542;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 100 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_15000/sam [2m [0m
|
| 101 |
+
[2;36m [0m pler.bin [2m [0m
|
| 102 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=693342;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=885391;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 103 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_15000/sam [2m [0m
|
| 104 |
+
[2;36m [0m pler_1.bin [2m [0m
|
| 105 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=558902;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=434201;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 106 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_15000/ran [2m [0m
|
| 107 |
+
[2;36m [0m dom_states_0.pkl [2m [0m
|
| 108 |
+
Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_15000
|
| 109 |
+
[2;36m[17:39:21 04/08][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=196292;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=360779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 110 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_20000 [2m [0m
|
| 111 |
+
[2;36m[17:39:22 04/08][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=818149;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=854583;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 112 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_20000/mod [2m [0m
|
| 113 |
+
[2;36m [0m el.safetensors [2m [0m
|
| 114 |
+
[2;36m[17:39:23 04/08][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=341614;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=544562;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 115 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_20000/opt [2m [0m
|
| 116 |
+
[2;36m [0m imizer.bin [2m [0m
|
| 117 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=835298;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=550186;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 118 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_20000/sch [2m [0m
|
| 119 |
+
[2;36m [0m eduler.bin [2m [0m
|
| 120 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=889056;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=224742;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 121 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_20000/sam [2m [0m
|
| 122 |
+
[2;36m [0m pler.bin [2m [0m
|
| 123 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=921300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=476172;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 124 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_20000/sam [2m [0m
|
| 125 |
+
[2;36m [0m pler_1.bin [2m [0m
|
| 126 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=821364;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=874045;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 127 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_20000/ran [2m [0m
|
| 128 |
+
[2;36m [0m dom_states_0.pkl [2m [0m
|
| 129 |
+
Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_20000
|
| 130 |
+
[2;36m[18:06:10 04/08][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=28505;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=369046;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 131 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_25000 [2m [0m
|
| 132 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=720536;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=600133;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 133 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_25000/mod [2m [0m
|
| 134 |
+
[2;36m [0m el.safetensors [2m [0m
|
| 135 |
+
[2;36m[18:06:11 04/08][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=650939;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=750793;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 136 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_25000/opt [2m [0m
|
| 137 |
+
[2;36m [0m imizer.bin [2m [0m
|
| 138 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=766478;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=224851;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 139 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_25000/sch [2m [0m
|
| 140 |
+
[2;36m [0m eduler.bin [2m [0m
|
| 141 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=702868;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=582729;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 142 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_25000/sam [2m [0m
|
| 143 |
+
[2;36m [0m pler.bin [2m [0m
|
| 144 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=101114;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=457234;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 145 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_25000/sam [2m [0m
|
| 146 |
+
[2;36m [0m pler_1.bin [2m [0m
|
| 147 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=795987;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=373558;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 148 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_25000/ran [2m [0m
|
| 149 |
+
[2;36m [0m dom_states_0.pkl [2m [0m
|
| 150 |
+
Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_25000
|
| 151 |
+
[2;36m[18:37:31 04/08][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=660693;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=111484;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 152 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_30000 [2m [0m
|
| 153 |
+
[2;36m[18:37:32 04/08][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=878068;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=67019;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 154 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_30000/mod [2m [0m
|
| 155 |
+
[2;36m [0m el.safetensors [2m [0m
|
| 156 |
+
[2;36m[18:37:33 04/08][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=273529;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=268565;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 157 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_30000/opt [2m [0m
|
| 158 |
+
[2;36m [0m imizer.bin [2m [0m
|
| 159 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=941677;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=948866;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 160 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_30000/sch [2m [0m
|
| 161 |
+
[2;36m [0m eduler.bin [2m [0m
|
| 162 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=60095;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=101135;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 163 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_30000/sam [2m [0m
|
| 164 |
+
[2;36m [0m pler.bin [2m [0m
|
| 165 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=98429;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=719141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 166 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_30000/sam [2m [0m
|
| 167 |
+
[2;36m [0m pler_1.bin [2m [0m
|
| 168 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=367323;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=330274;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 169 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_30000/ran [2m [0m
|
| 170 |
+
[2;36m [0m dom_states_0.pkl [2m [0m
|
| 171 |
+
Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_30000
|
| 172 |
+
[2;36m[19:06:07 04/08][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=686520;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=748910;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 173 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_35000 [2m [0m
|
| 174 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=331416;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=637309;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 175 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_35000/mod [2m [0m
|
| 176 |
+
[2;36m [0m el.safetensors [2m [0m
|
| 177 |
+
[2;36m[19:06:08 04/08][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=708752;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=921072;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 178 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_35000/opt [2m [0m
|
| 179 |
+
[2;36m [0m imizer.bin [2m [0m
|
| 180 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=525611;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=714503;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 181 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_35000/sch [2m [0m
|
| 182 |
+
[2;36m [0m eduler.bin [2m [0m
|
| 183 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=993068;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=856493;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 184 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_35000/sam [2m [0m
|
| 185 |
+
[2;36m [0m pler.bin [2m [0m
|
| 186 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=457956;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=921087;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 187 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_35000/sam [2m [0m
|
| 188 |
+
[2;36m [0m pler_1.bin [2m [0m
|
| 189 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=303621;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=762447;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 190 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_35000/ran [2m [0m
|
| 191 |
+
[2;36m [0m dom_states_0.pkl [2m [0m
|
| 192 |
+
Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_35000
|
| 193 |
+
[2;36m[19:34:21 04/08][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=311366;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=805194;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 194 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_40000 [2m [0m
|
| 195 |
+
[2;36m[19:34:23 04/08][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=21254;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=568492;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 196 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_40000/mod [2m [0m
|
| 197 |
+
[2;36m [0m el.safetensors [2m [0m
|
| 198 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=756734;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=118067;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 199 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_40000/opt [2m [0m
|
| 200 |
+
[2;36m [0m imizer.bin [2m [0m
|
| 201 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=39131;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=131787;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 202 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_40000/sch [2m [0m
|
| 203 |
+
[2;36m [0m eduler.bin [2m [0m
|
| 204 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=687655;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=316922;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 205 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_40000/sam [2m [0m
|
| 206 |
+
[2;36m [0m pler.bin [2m [0m
|
| 207 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=373263;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=435471;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 208 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_40000/sam [2m [0m
|
| 209 |
+
[2;36m [0m pler_1.bin [2m [0m
|
| 210 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=441435;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=694779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 211 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_40000/ran [2m [0m
|
| 212 |
+
[2;36m [0m dom_states_0.pkl [2m [0m
|
| 213 |
+
Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000
|
| 214 |
+
Training has reached maximum steps.
|
| 215 |
+
[2;36m[19:34:30 04/08][0m[2;36m [0m[34mINFO [0m | >> Saving current state to ]8;id=569300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=29797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\[2m3260[0m]8;;\
|
| 216 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_39999 [2m [0m
|
| 217 |
+
[2;36m[19:34:31 04/08][0m[2;36m [0m[34mINFO [0m | >> Model weights saved in ]8;id=716966;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=581852;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\[2m108[0m]8;;\
|
| 218 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_39999/mod [2m [0m
|
| 219 |
+
[2;36m [0m el.safetensors [2m [0m
|
| 220 |
+
[2;36m[19:34:32 04/08][0m[2;36m [0m[34mINFO [0m | >> Optimizer state saved in ]8;id=154857;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=239343;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 221 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_39999/opt [2m [0m
|
| 222 |
+
[2;36m [0m imizer.bin [2m [0m
|
| 223 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Scheduler state saved in ]8;id=502103;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=438178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 224 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_39999/sch [2m [0m
|
| 225 |
+
[2;36m [0m eduler.bin [2m [0m
|
| 226 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m0[0m saved in ]8;id=657224;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=622487;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 227 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_39999/sam [2m [0m
|
| 228 |
+
[2;36m [0m pler.bin [2m [0m
|
| 229 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Sampler state for dataloader [1;36m1[0m saved in ]8;id=39855;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=681797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\[2m139[0m]8;;\
|
| 230 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_39999/sam [2m [0m
|
| 231 |
+
[2;36m [0m pler_1.bin [2m [0m
|
| 232 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Random states saved in ]8;id=278106;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=357263;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\[2m170[0m]8;;\
|
| 233 |
+
[2;36m [0m .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604081550[0m/checkpoints/ckpt_39999/ran [2m [0m
|
| 234 |
+
[2;36m [0m dom_states_0.pkl [2m [0m
|
| 235 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Finalized ACT Trainer. Epoch losses: [1m[[0m[1;36m0.0[0m, [1;36m1.9138156175613403[0m, [1;36m0.7621236443519592[0m, [1;36m0.3085829019546509[0m, [1;36m0.21359018981456757[0m, ]8;id=58988;file:///data/jliu/psi/src/psi/trainers/act_g1.py\[2mact_g1.py[0m]8;;\[2m:[0m]8;id=689824;file:///data/jliu/psi/src/psi/trainers/act_g1.py#331\[2m331[0m]8;;\
|
| 236 |
+
[2;36m [0m [1;36m0.15654177963733673[0m[1m][0m [2m [0m
|
| 237 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Happy Ending! ]8;id=61128;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=232614;file:///data/jliu/psi/scripts/train.py#310\[2m310[0m]8;;\
|
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/requirements.txt
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
tifffile==2025.5.10
|
| 2 |
+
mpmath==1.3.0
|
| 3 |
+
pynput==1.8.1
|
| 4 |
+
exceptiongroup==1.3.0
|
| 5 |
+
sympy==1.14.0
|
| 6 |
+
decorator==5.2.1
|
| 7 |
+
aiohttp-cors==0.8.1
|
| 8 |
+
soupsieve==2.8
|
| 9 |
+
Farama-Notifications==0.0.4
|
| 10 |
+
numpydantic==1.6.7
|
| 11 |
+
uvicorn==0.38.0
|
| 12 |
+
waterbear==2.6.8
|
| 13 |
+
comm==0.2.3
|
| 14 |
+
cmake==4.2.3
|
| 15 |
+
xxhash==3.5.0
|
| 16 |
+
parso==0.8.6
|
| 17 |
+
nvidia-cusparse-cu12==12.5.4.2
|
| 18 |
+
annotated-doc==0.0.4
|
| 19 |
+
jsonlines==4.0.0
|
| 20 |
+
nvidia-cuda-runtime-cu12==12.6.77
|
| 21 |
+
pydantic==2.10.6
|
| 22 |
+
multidict==6.6.4
|
| 23 |
+
shtab==1.7.2
|
| 24 |
+
nvidia-cuda-nvrtc-cu12==12.6.77
|
| 25 |
+
matplotlib-inline==0.2.1
|
| 26 |
+
pycollada==0.9.2
|
| 27 |
+
pure_eval==0.2.3
|
| 28 |
+
beautifulsoup4==4.14.2
|
| 29 |
+
msgspec==0.19.0
|
| 30 |
+
huggingface-hub==0.35.3
|
| 31 |
+
pydantic_core==2.27.2
|
| 32 |
+
pytz==2025.2
|
| 33 |
+
gymnasium==1.2.3
|
| 34 |
+
attrs==25.3.0
|
| 35 |
+
requests==2.32.5
|
| 36 |
+
hf_transfer==0.1.9
|
| 37 |
+
einx==0.3.0
|
| 38 |
+
scipy==1.15.3
|
| 39 |
+
transforms3d==0.4.2
|
| 40 |
+
pyzmq==27.1.0
|
| 41 |
+
aiohappyeyeballs==2.6.1
|
| 42 |
+
six==1.17.0
|
| 43 |
+
expandvars==1.1.2
|
| 44 |
+
prompt_toolkit==3.0.52
|
| 45 |
+
wcwidth==0.6.0
|
| 46 |
+
stack-data==0.6.3
|
| 47 |
+
ruamel.yaml.clib==0.2.14
|
| 48 |
+
multiprocess==0.70.16
|
| 49 |
+
vhacdx==0.0.9
|
| 50 |
+
tokenizers==0.22.2
|
| 51 |
+
aiosignal==1.4.0
|
| 52 |
+
itsdangerous==2.2.0
|
| 53 |
+
torchvision==0.22.0
|
| 54 |
+
nvidia-cublas-cu12==12.6.4.1
|
| 55 |
+
httpx==0.28.1
|
| 56 |
+
platformdirs==4.4.0
|
| 57 |
+
sentry-sdk==2.39.0
|
| 58 |
+
plotly==6.2.0
|
| 59 |
+
nvidia-cudnn-cu12==9.5.1.17
|
| 60 |
+
jsonschema==4.25.1
|
| 61 |
+
fastapi==0.119.1
|
| 62 |
+
fsspec==2025.3.0
|
| 63 |
+
MarkupSafe==3.0.3
|
| 64 |
+
pydantic-yaml==1.6.0
|
| 65 |
+
h11==0.16.0
|
| 66 |
+
typer==0.24.1
|
| 67 |
+
mypy_extensions==1.1.0
|
| 68 |
+
nest-asyncio==1.6.0
|
| 69 |
+
h5py==3.14.0
|
| 70 |
+
python-xlib==0.33
|
| 71 |
+
lazy_loader==0.4
|
| 72 |
+
einops==0.8.1
|
| 73 |
+
albumentations==1.4.18
|
| 74 |
+
params_proto==2.13.2
|
| 75 |
+
psutil==7.1.0
|
| 76 |
+
starlette==0.48.0
|
| 77 |
+
anyio==4.11.0
|
| 78 |
+
gdown==5.2.0
|
| 79 |
+
charset-normalizer==3.4.3
|
| 80 |
+
tyro==0.9.32
|
| 81 |
+
filelock==3.19.1
|
| 82 |
+
websockets==15.0.1
|
| 83 |
+
nvidia-cuda-cupti-cu12==12.6.80
|
| 84 |
+
python-dotenv==1.2.1
|
| 85 |
+
orderly-set==5.5.0
|
| 86 |
+
inquirerpy==0.3.4
|
| 87 |
+
pexpect==4.9.0
|
| 88 |
+
ipython==8.38.0
|
| 89 |
+
urllib3==2.5.0
|
| 90 |
+
diffusers==0.37.0
|
| 91 |
+
PyYAML==6.0.3
|
| 92 |
+
antlr4-python3-runtime==4.9.3
|
| 93 |
+
mdurl==0.1.2
|
| 94 |
+
omegaconf==2.3.0
|
| 95 |
+
rerun-sdk==0.22.1
|
| 96 |
+
draccus==0.10.0
|
| 97 |
+
sentencepiece==0.2.1
|
| 98 |
+
referencing==0.37.0
|
| 99 |
+
docstring_parser==0.17.0
|
| 100 |
+
protobuf==6.33.5
|
| 101 |
+
wandb==0.25.1
|
| 102 |
+
numpy==1.26.4
|
| 103 |
+
GitPython==3.1.45
|
| 104 |
+
opencv-python-headless==4.11.0.86
|
| 105 |
+
yourdfpy==0.0.58
|
| 106 |
+
async-timeout==4.0.3
|
| 107 |
+
shapely==2.1.2
|
| 108 |
+
frozenlist==1.7.0
|
| 109 |
+
simplejpeg==1.9.0
|
| 110 |
+
asttokens==3.0.1
|
| 111 |
+
Pygments==2.19.2
|
| 112 |
+
py-cpuinfo==9.0.0
|
| 113 |
+
rtree==1.4.1
|
| 114 |
+
jedi==0.19.2
|
| 115 |
+
gitdb==4.0.12
|
| 116 |
+
cloudpickle==3.1.1
|
| 117 |
+
deepspeed==0.17.1
|
| 118 |
+
executing==2.2.1
|
| 119 |
+
nvidia-nvjitlink-cu12==12.6.85
|
| 120 |
+
colorlog==6.10.1
|
| 121 |
+
qwen-vl-utils==0.0.14
|
| 122 |
+
Werkzeug==3.1.6
|
| 123 |
+
zipp==3.23.0
|
| 124 |
+
setuptools==80.9.0
|
| 125 |
+
albucore==0.0.17
|
| 126 |
+
transformers==4.57.0
|
| 127 |
+
imageio==2.34.2
|
| 128 |
+
frozendict==2.4.6
|
| 129 |
+
hjson==3.1.0
|
| 130 |
+
jsonschema-specifications==2025.9.1
|
| 131 |
+
rpds-py==0.28.0
|
| 132 |
+
tornado==6.5.5
|
| 133 |
+
blinker==1.9.0
|
| 134 |
+
accelerate==1.7.0
|
| 135 |
+
argparse==1.4.0
|
| 136 |
+
msgpack==1.1.1
|
| 137 |
+
smmap==5.0.2
|
| 138 |
+
nvidia-nvtx-cu12==12.6.77
|
| 139 |
+
packaging==25.0
|
| 140 |
+
embreex==2.17.7.post7
|
| 141 |
+
vuer==0.0.68
|
| 142 |
+
propcache==0.3.2
|
| 143 |
+
nvidia-cusolver-cu12==11.7.1.2
|
| 144 |
+
triton==3.3.0
|
| 145 |
+
pfzy==0.3.4
|
| 146 |
+
debugpy==1.8.20
|
| 147 |
+
torchcodec==0.4.0
|
| 148 |
+
dm-tree==0.1.8
|
| 149 |
+
ninja==1.13.0
|
| 150 |
+
ipykernel==7.2.0
|
| 151 |
+
traitlets==5.14.3
|
| 152 |
+
nvidia-cufile-cu12==1.11.1.6
|
| 153 |
+
datasets==3.6.0
|
| 154 |
+
nvidia-curand-cu12==10.3.7.77
|
| 155 |
+
flash_attn==2.7.4.post1
|
| 156 |
+
av==16.0.1
|
| 157 |
+
sniffio==1.3.1
|
| 158 |
+
Jinja2==3.1.6
|
| 159 |
+
narwhals==2.6.0
|
| 160 |
+
yarl==1.20.1
|
| 161 |
+
typeguard==4.4.4
|
| 162 |
+
termcolor==3.1.0
|
| 163 |
+
jupyter_client==8.8.0
|
| 164 |
+
importlib_metadata==8.7.1
|
| 165 |
+
eval_type_backport==0.2.2
|
| 166 |
+
certifi==2025.8.3
|
| 167 |
+
tzdata==2025.2
|
| 168 |
+
typing_extensions==4.15.0
|
| 169 |
+
viser==1.0.15
|
| 170 |
+
typing-inspect==0.9.0
|
| 171 |
+
annotated-types==0.7.0
|
| 172 |
+
svg.path==7.0
|
| 173 |
+
mergedeep==1.3.4
|
| 174 |
+
ptyprocess==0.7.0
|
| 175 |
+
psi==0.0.0
|
| 176 |
+
deepdiff==8.6.1
|
| 177 |
+
toml==0.10.2
|
| 178 |
+
click==8.3.0
|
| 179 |
+
dotenv==0.9.9
|
| 180 |
+
trimesh==4.8.3
|
| 181 |
+
nvidia-nccl-cu12==2.26.2
|
| 182 |
+
pyyaml-include==1.4.1
|
| 183 |
+
scikit-image==0.25.2
|
| 184 |
+
PySocks==1.7.1
|
| 185 |
+
aiohttp==3.10.5
|
| 186 |
+
opencv-python==4.11.0.86
|
| 187 |
+
nodeenv==1.9.1
|
| 188 |
+
lerobot==0.3.3
|
| 189 |
+
pillow==11.3.0
|
| 190 |
+
idna==3.10
|
| 191 |
+
Flask==3.1.3
|
| 192 |
+
killport==1.2.0
|
| 193 |
+
hf-xet==1.1.10
|
| 194 |
+
imageio-ffmpeg==0.6.0
|
| 195 |
+
httpcore==1.0.9
|
| 196 |
+
nvidia-cusparselt-cu12==0.6.3
|
| 197 |
+
networkx==3.4.2
|
| 198 |
+
ruamel.yaml==0.18.15
|
| 199 |
+
manifold3d==3.2.1
|
| 200 |
+
shellingham==1.5.4
|
| 201 |
+
nvidia-cufft-cu12==11.3.0.4
|
| 202 |
+
mapbox_earcut==1.0.3
|
| 203 |
+
peft==0.17.1
|
| 204 |
+
pyarrow==21.0.0
|
| 205 |
+
pyserial==3.5
|
| 206 |
+
torch==2.7.0
|
| 207 |
+
dill==0.3.8
|
| 208 |
+
markdown-it-py==4.0.0
|
| 209 |
+
tqdm==4.67.1
|
| 210 |
+
argcomplete==3.6.2
|
| 211 |
+
jupyter_core==5.9.1
|
| 212 |
+
pandas==2.3.3
|
| 213 |
+
lxml==6.0.2
|
| 214 |
+
safetensors==0.6.2
|
| 215 |
+
python-dateutil==2.9.0.post0
|
| 216 |
+
regex==2025.9.18
|
| 217 |
+
evdev==1.9.3
|
| 218 |
+
rich==14.1.0
|
| 219 |
+
psi==0.0.0
|