Add DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain from 91cf0fc7c553

.gitattributes

@@ -51,3 +51,4 @@ wandb/run-20250518_145405-bqr8y636/run-bqr8y636.wandb filter=lfs diff=lfs merge=
 DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250518_145405-bqr8y636/run-bqr8y636.wandb filter=lfs diff=lfs merge=lfs -text
 DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/run-fekph8dz.wandb filter=lfs diff=lfs merge=lfs -text
 DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250518_143255-5e56tibb/run-5e56tibb.wandb filter=lfs diff=lfs merge=lfs -text
+DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/run-5e56tibb.wandb filter=lfs diff=lfs merge=lfs -text

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/checkpoints/320000/pretrained_model/config.json

@@ -0,0 +1,107 @@
+{
+    "type": "diffusion",
+    "n_obs_steps": 2,
+    "normalization_mapping": {
+        "VISUAL": "MEAN_STD",
+        "STATE": "MIN_MAX",
+        "ACTION": "MIN_MAX"
+    },
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                6
+            ]
+        },
+        "observation.images.FrontCam": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.images.TopCam": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.images.WristCam": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                6
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "horizon": 16,
+    "n_action_steps": 8,
+    "drop_n_last_frames": 7,
+    "vision_backbone": "resnet50",
+    "crop_shape": [
+        224,
+        224
+    ],
+    "crop_is_random": false,
+    "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1",
+    "use_group_norm": false,
+    "spatial_softmax_num_keypoints": 32,
+    "use_separate_rgb_encoder_per_camera": false,
+    "down_dims": [
+        256,
+        512,
+        1024
+    ],
+    "kernel_size": 5,
+    "n_groups": 8,
+    "diffusion_step_embed_dim": 128,
+    "use_film_scale_modulation": true,
+    "noise_scheduler_type": "DDIM",
+    "num_train_timesteps": 100,
+    "beta_schedule": "squaredcos_cap_v2",
+    "beta_start": 0.0001,
+    "beta_end": 0.02,
+    "prediction_type": "epsilon",
+    "clip_sample": true,
+    "clip_sample_range": 1.0,
+    "num_inference_steps": 10,
+    "do_mask_loss_for_padding": false,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.95,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-06,
+    "scheduler_name": "cosine",
+    "scheduler_warmup_steps": 500,
+    "pre_resize_shape": [
+        224,
+        224
+    ],
+    "freeze_vision_backbone": false,
+    "use_ac": false,
+    "ac_threshold": 0.99,
+    "use_pcg": false,
+    "pcg_weight": 0.1,
+    "use_bid": false,
+    "bid_batch_size": 15,
+    "bid_mode_size": 3,
+    "bid_temporal_coherence": 0.5,
+    "weak_model_path": null,
+    "weak_model_config": null
+}

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/checkpoints/320000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:876e4757d896a63b5bde1d3bb0e36916337b19b8168083176f963175222200a8
+size 369241872

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/checkpoints/320000/pretrained_model/train_config.json

@@ -0,0 +1,217 @@
+{
+    "dataset": {
+        "repo_id": "shylee/so100_pengripG",
+        "root": "/SSD/LSY/lerobot_dataset/pengripG",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "diffusion",
+        "n_obs_steps": 2,
+        "normalization_mapping": {
+            "VISUAL": "MEAN_STD",
+            "STATE": "MIN_MAX",
+            "ACTION": "MIN_MAX"
+        },
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    6
+                ]
+            },
+            "observation.images.FrontCam": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.images.TopCam": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.images.WristCam": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    6
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "horizon": 16,
+        "n_action_steps": 8,
+        "drop_n_last_frames": 7,
+        "vision_backbone": "resnet50",
+        "crop_shape": [
+            224,
+            224
+        ],
+        "crop_is_random": false,
+        "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1",
+        "use_group_norm": false,
+        "spatial_softmax_num_keypoints": 32,
+        "use_separate_rgb_encoder_per_camera": false,
+        "down_dims": [
+            256,
+            512,
+            1024
+        ],
+        "kernel_size": 5,
+        "n_groups": 8,
+        "diffusion_step_embed_dim": 128,
+        "use_film_scale_modulation": true,
+        "noise_scheduler_type": "DDIM",
+        "num_train_timesteps": 100,
+        "beta_schedule": "squaredcos_cap_v2",
+        "beta_start": 0.0001,
+        "beta_end": 0.02,
+        "prediction_type": "epsilon",
+        "clip_sample": true,
+        "clip_sample_range": 1.0,
+        "num_inference_steps": 10,
+        "do_mask_loss_for_padding": false,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.95,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-06,
+        "scheduler_name": "cosine",
+        "scheduler_warmup_steps": 500,
+        "pre_resize_shape": [
+            224,
+            224
+        ],
+        "freeze_vision_backbone": false,
+        "use_ac": false,
+        "ac_threshold": 0.99,
+        "use_pcg": false,
+        "pcg_weight": 0.1,
+        "use_bid": false,
+        "bid_batch_size": 15,
+        "bid_mode_size": 3,
+        "bid_temporal_coherence": 0.5,
+        "weak_model_path": null,
+        "weak_model_config": null
+    },
+    "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain",
+    "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain",
+    "resume": true,
+    "seed": 1000,
+    "num_workers": 4,
+    "batch_size": 16,
+    "steps": 320000,
+    "eval_freq": 20000,
+    "log_freq": 200,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adam",
+        "lr": 0.0001,
+        "weight_decay": 1e-06,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "5e56tibb",
+        "mode": null
+    }
+}

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/checkpoints/320000/training_state/optimizer_param_groups.json

@@ -0,0 +1,331 @@
+[
+    {
+        "lr": 0.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-06,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211,
+            212,
+            213,
+            214,
+            215,
+            216,
+            217,
+            218,
+            219,
+            220,
+            221,
+            222,
+            223,
+            224,
+            225,
+            226,
+            227,
+            228,
+            229,
+            230,
+            231,
+            232,
+            233,
+            234,
+            235,
+            236,
+            237,
+            238,
+            239,
+            240,
+            241,
+            242,
+            243,
+            244,
+            245,
+            246,
+            247,
+            248,
+            249,
+            250,
+            251,
+            252,
+            253,
+            254,
+            255,
+            256,
+            257,
+            258,
+            259,
+            260,
+            261,
+            262,
+            263,
+            264,
+            265,
+            266,
+            267,
+            268,
+            269,
+            270,
+            271,
+            272,
+            273,
+            274,
+            275,
+            276,
+            277,
+            278,
+            279,
+            280,
+            281,
+            282,
+            283,
+            284,
+            285,
+            286,
+            287,
+            288,
+            289,
+            290,
+            291,
+            292,
+            293,
+            294,
+            295,
+            296,
+            297,
+            298,
+            299,
+            300,
+            301,
+            302,
+            303,
+            304,
+            305,
+            306,
+            307,
+            308,
+            309,
+            310
+        ]
+    }
+]

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/checkpoints/320000/training_state/optimizer_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d82cae1786bb5912dce69f9e6dbbb18a10760c78955bd5a74dea5300242bd8c8
+size 738026076

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/checkpoints/320000/training_state/rng_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a05c17f21b1a33f49a52a4c14a899da62bea9c5c0e901b3c3e35fdf58f6bd4d
+size 15708

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/checkpoints/320000/training_state/scheduler_state.json

@@ -0,0 +1,15 @@
+{
+    "base_lrs": [
+        0.0001
+    ],
+    "last_epoch": 320000,
+    "verbose": false,
+    "_step_count": 320001,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        0.0
+    ],
+    "lr_lambdas": [
+        null
+    ]
+}

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/checkpoints/320000/training_state/training_step.json

@@ -0,0 +1,3 @@
+{
+    "step": 320000
+}

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/debug-internal.log

@@ -1,17 +1,393 @@
-{"time":"2025-05-18T14:32:55.638534229Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250518_143255-5e56tibb/logs/debug-core.log"}
-{"time":"2025-05-18T14:32:56.101504943Z","level":"INFO","msg":"created new stream","id":"5e56tibb"}
-{"time":"2025-05-18T14:32:56.101628421Z","level":"INFO","msg":"stream: started","id":"5e56tibb"}
-{"time":"2025-05-18T14:32:56.101694211Z","level":"INFO","msg":"writer: Do: started","stream_id":"5e56tibb"}
-{"time":"2025-05-18T14:32:56.101756719Z","level":"INFO","msg":"sender: started","stream_id":"5e56tibb"}
-{"time":"2025-05-18T14:32:56.101859804Z","level":"INFO","msg":"handler: started","stream_id":"5e56tibb"}
-{"time":"2025-05-18T14:32:56.491309926Z","level":"INFO","msg":"Starting system monitor"}
-{"time":"2025-05-19T10:13:32.113609402Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/5e56tibb/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:36951->141.223.7.51:53: i/o timeout"}
-{"time":"2025-05-19T11:29:42.313813753Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/5e56tibb/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
-{"time":"2025-05-19T14:27:21.513094205Z","level":"INFO","msg":"stream: closing","id":"5e56tibb"}
-{"time":"2025-05-19T14:27:21.513138694Z","level":"INFO","msg":"Stopping system monitor"}
-{"time":"2025-05-19T14:27:21.513215868Z","level":"INFO","msg":"Stopped system monitor"}
-{"time":"2025-05-19T14:27:22.995893989Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-05-19T14:27:23.291673013Z","level":"INFO","msg":"handler: closed","stream_id":"5e56tibb"}
-{"time":"2025-05-19T14:27:23.291768066Z","level":"INFO","msg":"sender: closed","stream_id":"5e56tibb"}
-{"time":"2025-05-19T14:27:23.291759057Z","level":"INFO","msg":"writer: Close: closed","stream_id":"5e56tibb"}
-{"time":"2025-05-19T14:27:23.291970605Z","level":"INFO","msg":"stream: closed","id":"5e56tibb"}
+{"time":"2025-05-19T15:01:38.775891619Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/logs/debug-core.log"}
+{"time":"2025-05-19T15:01:39.112865708Z","level":"INFO","msg":"created new stream","id":"5e56tibb"}
+{"time":"2025-05-19T15:01:39.112931498Z","level":"INFO","msg":"stream: started","id":"5e56tibb"}
+{"time":"2025-05-19T15:01:39.112972146Z","level":"INFO","msg":"sender: started","stream_id":"5e56tibb"}
+{"time":"2025-05-19T15:01:39.113088081Z","level":"INFO","msg":"handler: started","stream_id":"5e56tibb"}
+{"time":"2025-05-19T15:01:39.113144792Z","level":"INFO","msg":"writer: Do: started","stream_id":"5e56tibb"}
+{"time":"2025-05-19T15:01:40.811234696Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-05-19T15:02:48.970606405Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970718569Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970728976Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970738404Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970746925Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970754467Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970762569Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970769204Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970778213Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:03:49.51029375Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510388803Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510402073Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510413876Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510424841Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510437063Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510447469Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510458365Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510468771Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:04:49.410214863Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410272062Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.41028205Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410291548Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410300488Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410308589Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410330729Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410339249Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.41034784Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:05:45.346295957Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346378718Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346388635Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346396667Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346404699Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346411334Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346426978Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346435499Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.3464436Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:06:43.322171399Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322258839Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322268338Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322276858Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.32228489Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322293899Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322300953Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322308566Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.32231562Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:07:40.490220028Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490282047Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490292872Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490301882Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490309913Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490318434Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490326466Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490334497Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490342599Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:08:38.494324851Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494451681Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494463973Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494474869Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494484297Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494493726Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494504621Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494513211Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494523617Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:09:35.618275757Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618372696Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618387363Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618399236Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618411458Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618424797Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618436531Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618448404Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618460207Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:10:32.982355474Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982486984Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982499276Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982508705Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982518203Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982527632Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.98253713Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982546628Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982555149Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:11:31.670364782Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670516127Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670528419Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670564876Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670575282Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670586178Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670597562Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670606501Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670615511Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:12:28.781744515Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.78179934Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.781809258Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.781895441Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782026463Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782112087Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782210493Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782306594Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782390333Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:13:26.696961702Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.69703273Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697092305Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697202513Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697300919Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697417343Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697481178Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697631615Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697674707Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:14:24.710189485Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710262818Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710272805Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710282304Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710290754Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710298367Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710317224Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710325256Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710333287Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:15:23.066204083Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066264635Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066279721Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066294457Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066307168Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066320019Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.06633273Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066345511Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066357314Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:16:22.030202312Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030301207Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030311124Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030321042Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030329562Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030338572Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030346673Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030356102Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.03036553Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:17:20.850184668Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850264146Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850272667Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850281188Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850289219Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850296343Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850303886Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850311498Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850318064Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:18:19.162114604Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.16216894Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162180324Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162189333Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162198762Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162208679Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162216711Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162226698Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162236127Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:19:18.598200374Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598296963Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598307789Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598316798Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.59832483Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598334328Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.59834236Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598351369Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.59835982Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:20:18.890268849Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.89034742Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890356848Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.89036495Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890372982Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890383318Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.89039135Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890400848Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890409369Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:21:27.978855062Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.978924554Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979014928Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979107676Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979200914Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979292684Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979380195Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979484746Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979581755Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:22:33.850208001Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850308782Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850320585Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850332458Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850340978Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850349499Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850358508Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850366959Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850374502Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:23:40.554189834Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554286843Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554295852Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.55430577Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554314221Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554322252Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554330354Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554339293Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554347814Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:24:46.958238203Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958303993Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958315796Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958326622Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.95833619Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958345618Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958368316Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958376348Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958385846Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:25:51.910176789Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.910250122Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.91026004Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.910269538Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.9102775Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.91028602Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.91029503Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.91030355Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.910311582Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:26:56.877751832Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.877905132Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.877916936Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.877927272Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878013944Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878177092Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878222978Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878298266Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878370202Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:28:00.946443925Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.946623695Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.946669161Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.946785516Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.946877287Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.947022625Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.947118098Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.947227887Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.947368896Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:29:03.382273603Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382338416Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382349241Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.38235825Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382366771Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382376688Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382386117Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382395196Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382405114Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:30:05.886211776Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886300683Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886311579Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886320588Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886329039Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886453076Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886544358Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886555672Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886566637Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:31:07.938290568Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938380942Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938392326Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938401335Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938410345Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938418865Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938426827Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938435348Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938443868Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:32:10.554145176Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554221302Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554244978Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554253988Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554262928Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554271448Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554280946Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554290375Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554299384Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:33:13.330208354Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330282175Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330292163Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330301661Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330310112Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330319121Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.33032855Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.33033707Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.33034608Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:34:15.038319398Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038438198Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038450909Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038460896Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038471302Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038482616Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038493581Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038504407Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038514813Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:35:18.042214537Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042282702Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042294505Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042303933Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042313432Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042341368Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042350797Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042360295Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042368815Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:36:20.478114919Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478200474Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478211858Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478221287Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478231274Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478241191Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478251109Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478260118Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478269547Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:37:23.530156874Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530242499Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530253813Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530263731Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530271832Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530279864Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530287895Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530295927Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530303959Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:38:25.694242948Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694304059Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694313976Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694323894Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694333811Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.69434324Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694352249Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694362236Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.69438689Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:39:28.571758776Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.571865214Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.571936172Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572038838Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572129701Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572211554Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572297109Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572391813Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572472689Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:40:30.590914294Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.590993773Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591096439Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591179759Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591271041Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591374615Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591458843Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591561997Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591641895Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:41:38.278181827Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278239516Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278249922Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278258932Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278267452Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278276462Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278284912Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278292944Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278300557Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:42:45.914075361Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914133119Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914156795Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914165316Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914173836Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914182776Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914191297Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914198839Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914206941Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:43:53.270063574Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270127479Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270138793Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270146824Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270154856Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270163377Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270171408Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.27017951Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270188031Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:45:00.713439117Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.713600938Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.713724417Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.713856416Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.713973678Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.714098623Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.714170071Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.714298717Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.714359828Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T16:47:15.306554546Z","level":"INFO","msg":"stream: closing","id":"5e56tibb"}
+{"time":"2025-05-19T16:47:15.306584367Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-05-19T16:47:15.306645827Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-05-19T16:47:34.765612049Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-05-19T16:47:35.023526244Z","level":"INFO","msg":"handler: closed","stream_id":"5e56tibb"}
+{"time":"2025-05-19T16:47:35.023574923Z","level":"INFO","msg":"writer: Close: closed","stream_id":"5e56tibb"}
+{"time":"2025-05-19T16:47:35.023585399Z","level":"INFO","msg":"sender: closed","stream_id":"5e56tibb"}
+{"time":"2025-05-19T16:47:35.02365964Z","level":"INFO","msg":"stream: closed","id":"5e56tibb"}

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/debug.log

@@ -1,23 +1,24 @@
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_setup.py:_flush():67] Configure stats pid to 6377
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_setup.py:_flush():67] Loading settings from environment variables
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250518_143255-5e56tibb/logs/debug.log
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250518_143255-5e56tibb/logs/debug-internal.log
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_init.py:init():781] calling init triggers
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
-config: {'dataset': {'repo_id': 'shylee/so100_pengripG', 'root': '/SSD/LSY/lerobot_dataset/pengripG', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MIN_MAX: 'MIN_MAX'>, 'ACTION': <NormalizationMode.MIN_MAX: 'MIN_MAX'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ac': False, 'ac_threshold': 0.99, 'use_pcg': False, 'pcg_weight': 0.1, 'use_bid': False, 'bid_batch_size': 15, 'bid_mode_size': 3, 'bid_temporal_coherence': 0.5, 'weak_model_path': None, 'weak_model_config': None}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain', 'job_name': 'DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 320000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_init.py:init():809] starting backend
-2025-05-18 14:32:55,602 INFO    MainThread:6377 [wandb_init.py:init():813] sending inform_init request
-2025-05-18 14:32:55,631 INFO    MainThread:6377 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2025-05-18 14:32:55,631 INFO    MainThread:6377 [wandb_init.py:init():823] backend started and connected
-2025-05-18 14:32:55,768 INFO    MainThread:6377 [wandb_init.py:init():915] updated telemetry
-2025-05-18 14:32:55,822 INFO    MainThread:6377 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
-2025-05-18 14:32:56,483 INFO    MainThread:6377 [wandb_init.py:init():1014] starting run threads in backend
-2025-05-18 14:32:56,631 INFO    MainThread:6377 [wandb_run.py:_console_start():2454] atexit reg
-2025-05-18 14:32:56,631 INFO    MainThread:6377 [wandb_run.py:_redirect():2306] redirect: wrap_raw
-2025-05-18 14:32:56,631 INFO    MainThread:6377 [wandb_run.py:_redirect():2371] Wrapping output streams.
-2025-05-18 14:32:56,631 INFO    MainThread:6377 [wandb_run.py:_redirect():2394] Redirects installed.
-2025-05-18 14:32:56,633 INFO    MainThread:6377 [wandb_init.py:init():1056] run started, returning control to user process
-2025-05-19 14:27:21,511 INFO    MsgRouterThr:6377 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Configure stats pid to 827647
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/logs/debug.log
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/logs/debug-internal.log
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:init():781] calling init triggers
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'shylee/so100_pengripG', 'root': '/SSD/LSY/lerobot_dataset/pengripG', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MIN_MAX: 'MIN_MAX'>, 'ACTION': <NormalizationMode.MIN_MAX: 'MIN_MAX'>}, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [6]}, 'observation.images.FrontCam': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 480, 640]}, 'observation.images.TopCam': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 480, 640]}, 'observation.images.WristCam': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 480, 640]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ac': False, 'ac_threshold': 0.99, 'use_pcg': False, 'pcg_weight': 0.1, 'use_bid': False, 'bid_batch_size': 15, 'bid_mode_size': 3, 'bid_temporal_coherence': 0.5, 'weak_model_path': None, 'weak_model_config': None}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain', 'job_name': 'DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain', 'resume': True, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 320000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '5e56tibb', 'mode': None}, '_wandb': {}}
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:init():809] starting backend
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:init():813] sending inform_init request
+2025-05-19 15:01:38,773 INFO    MainThread:827647 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-05-19 15:01:38,774 INFO    MainThread:827647 [wandb_init.py:init():823] backend started and connected
+2025-05-19 15:01:39,150 INFO    MainThread:827647 [wandb_init.py:init():915] updated telemetry
+2025-05-19 15:01:39,842 INFO    MainThread:827647 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
+2025-05-19 15:01:40,769 INFO    MainThread:827647 [wandb_init.py:init():1009] run resumed
+2025-05-19 15:01:40,772 INFO    MainThread:827647 [wandb_init.py:init():1014] starting run threads in backend
+2025-05-19 15:01:40,956 INFO    MainThread:827647 [wandb_run.py:_console_start():2454] atexit reg
+2025-05-19 15:01:40,956 INFO    MainThread:827647 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-05-19 15:01:40,956 INFO    MainThread:827647 [wandb_run.py:_redirect():2371] Wrapping output streams.
+2025-05-19 15:01:40,957 INFO    MainThread:827647 [wandb_run.py:_redirect():2394] Redirects installed.
+2025-05-19 15:01:40,958 INFO    MainThread:827647 [wandb_init.py:init():1056] run started, returning control to user process
+2025-05-19 16:47:15,305 INFO    MsgRouterThr:827647 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/files/config.yaml

@@ -0,0 +1,228 @@
+_wandb:
+    value:
+        cli_version: 0.19.9
+        m: []
+        python_version: 3.10.17
+        t:
+            "1":
+                - 1
+                - 41
+                - 49
+                - 51
+                - 55
+            "2":
+                - 1
+                - 41
+                - 49
+                - 51
+                - 55
+                - 83
+            "3":
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 23
+                - 55
+                - 61
+                - 62
+            "4": 3.10.17
+            "5": 0.19.9
+            "8":
+                - 5
+            "12": 0.19.9
+            "13": linux-x86_64
+batch_size:
+    value: 16
+dataset:
+    value:
+        episodes: null
+        image_transforms:
+            enable: false
+            max_num_transforms: 3
+            random_order: false
+            tfs:
+                brightness:
+                    kwargs:
+                        brightness:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                contrast:
+                    kwargs:
+                        contrast:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                hue:
+                    kwargs:
+                        hue:
+                            - -0.05
+                            - 0.05
+                    type: ColorJitter
+                    weight: 1
+                saturation:
+                    kwargs:
+                        saturation:
+                            - 0.5
+                            - 1.5
+                    type: ColorJitter
+                    weight: 1
+                sharpness:
+                    kwargs:
+                        sharpness:
+                            - 0.5
+                            - 1.5
+                    type: SharpnessJitter
+                    weight: 1
+        repo_id: shylee/so100_pengripG
+        revision: null
+        root: /SSD/LSY/lerobot_dataset/pengripG
+        use_imagenet_stats: true
+        video_backend: torchcodec
+env:
+    value: null
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        use_async_envs: false
+eval_freq:
+    value: 20000
+job_name:
+    value: DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain
+log_freq:
+    value: 200
+num_workers:
+    value: 4
+optimizer:
+    value:
+        betas:
+            - 0.95
+            - 0.999
+        eps: 1e-08
+        grad_clip_norm: 10
+        lr: 0.0001
+        type: adam
+        weight_decay: 1e-06
+output_dir:
+    value: /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain
+policy:
+    value:
+        ac_threshold: 0.99
+        beta_end: 0.02
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        bid_batch_size: 15
+        bid_mode_size: 3
+        bid_temporal_coherence: 0.5
+        clip_sample: true
+        clip_sample_range: 1
+        crop_is_random: false
+        crop_shape:
+            - 224
+            - 224
+        device: cuda
+        diffusion_step_embed_dim: 128
+        do_mask_loss_for_padding: false
+        down_dims:
+            - 256
+            - 512
+            - 1024
+        drop_n_last_frames: 7
+        freeze_vision_backbone: false
+        horizon: 16
+        input_features:
+            observation.images.FrontCam:
+                shape:
+                    - 3
+                    - 480
+                    - 640
+                type: VISUAL
+            observation.images.TopCam:
+                shape:
+                    - 3
+                    - 480
+                    - 640
+                type: VISUAL
+            observation.images.WristCam:
+                shape:
+                    - 3
+                    - 480
+                    - 640
+                type: VISUAL
+            observation.state:
+                shape:
+                    - 6
+                type: STATE
+        kernel_size: 5
+        n_action_steps: 8
+        n_groups: 8
+        n_obs_steps: 2
+        noise_scheduler_type: DDIM
+        normalization_mapping:
+            ACTION: MIN_MAX
+            STATE: MIN_MAX
+            VISUAL: MEAN_STD
+        num_inference_steps: 10
+        num_train_timesteps: 100
+        optimizer_betas:
+            - 0.95
+            - 0.999
+        optimizer_eps: 1e-08
+        optimizer_lr: 0.0001
+        optimizer_weight_decay: 1e-06
+        output_features:
+            action:
+                shape:
+                    - 6
+                type: ACTION
+        pcg_weight: 0.1
+        pre_resize_shape:
+            - 224
+            - 224
+        prediction_type: epsilon
+        pretrained_backbone_weights: ResNet50_Weights.IMAGENET1K_V1
+        scheduler_name: cosine
+        scheduler_warmup_steps: 500
+        spatial_softmax_num_keypoints: 32
+        type: diffusion
+        use_ac: false
+        use_amp: false
+        use_bid: false
+        use_film_scale_modulation: true
+        use_group_norm: false
+        use_pcg: false
+        use_separate_rgb_encoder_per_camera: false
+        vision_backbone: resnet50
+        weak_model_config: null
+        weak_model_path: null
+resume:
+    value: true
+save_checkpoint:
+    value: true
+save_freq:
+    value: 20000
+scheduler:
+    value:
+        name: cosine
+        num_warmup_steps: 500
+        type: diffuser
+seed:
+    value: 1000
+steps:
+    value: 320000
+use_policy_training_preset:
+    value: true
+wandb:
+    value:
+        disable_artifact: false
+        enable: true
+        entity: null
+        mode: null
+        notes: null
+        project: lerobot
+        run_id: 5e56tibb

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/files/output.log

@@ -0,0 +1,116 @@
+Logs will be synced with wandb.
+INFO 2025-05-19 15:01:40 ndb_utils.py:96 Track this run --> https://wandb.ai/marchmelo0923-postech/lerobot/runs/5e56tibb
+INFO 2025-05-19 15:01:41 ts/train.py:127 Creating dataset
+Resolving data files: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 300/300 [00:00<00:00, 214469.27it/s]
+INFO 2025-05-19 15:01:43 ts/train.py:138 Creating policy
+Loading weights from local directory
+INFO 2025-05-19 15:01:45 ts/train.py:144 Creating optimizer and scheduler
+INFO 2025-05-19 15:01:45 ts/train.py:156 Output dir: /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain
+INFO 2025-05-19 15:01:45 ts/train.py:159 cfg.steps=320000 (320K)
+INFO 2025-05-19 15:01:45 ts/train.py:160 dataset.num_frames=63435 (63K)
+INFO 2025-05-19 15:01:45 ts/train.py:161 dataset.num_episodes=300
+INFO 2025-05-19 15:01:45 ts/train.py:162 num_learnable_params=92242982 (92M)
+INFO 2025-05-19 15:01:45 ts/train.py:163 num_total_params=92243036 (92M)
+INFO 2025-05-19 15:01:45 ts/train.py:202 Start offline training on a fixed dataset
+INFO 2025-05-19 15:02:48 ts/train.py:232 step:300K smpl:5M ep:23K epch:75.72 loss:0.002 grdn:0.140 lr:9.5e-07 updt_s:0.244 data_s:0.071
+INFO 2025-05-19 15:03:49 ts/train.py:232 step:300K smpl:5M ep:23K epch:75.77 loss:0.002 grdn:0.144 lr:9.4e-07 updt_s:0.221 data_s:0.080
+INFO 2025-05-19 15:04:49 ts/train.py:232 step:301K smpl:5M ep:23K epch:75.82 loss:0.002 grdn:0.152 lr:9.2e-07 updt_s:0.222 data_s:0.076
+INFO 2025-05-19 15:05:45 ts/train.py:232 step:301K smpl:5M ep:23K epch:75.87 loss:0.002 grdn:0.136 lr:9.0e-07 updt_s:0.222 data_s:0.057
+INFO 2025-05-19 15:06:43 ts/train.py:232 step:301K smpl:5M ep:23K epch:75.92 loss:0.003 grdn:0.165 lr:8.8e-07 updt_s:0.222 data_s:0.067
+INFO 2025-05-19 15:07:40 ts/train.py:232 step:301K smpl:5M ep:23K epch:75.97 loss:0.003 grdn:0.157 lr:8.6e-07 updt_s:0.222 data_s:0.063
+INFO 2025-05-19 15:08:38 ts/train.py:232 step:301K smpl:5M ep:23K epch:76.02 loss:0.002 grdn:0.140 lr:8.4e-07 updt_s:0.222 data_s:0.068
+INFO 2025-05-19 15:09:35 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.07 loss:0.002 grdn:0.123 lr:8.2e-07 updt_s:0.222 data_s:0.063
+INFO 2025-05-19 15:10:32 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.12 loss:0.003 grdn:0.162 lr:8.1e-07 updt_s:0.223 data_s:0.063
+INFO 2025-05-19 15:11:31 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.17 loss:0.002 grdn:0.140 lr:7.9e-07 updt_s:0.222 data_s:0.071
+INFO 2025-05-19 15:12:28 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.22 loss:0.002 grdn:0.131 lr:7.7e-07 updt_s:0.222 data_s:0.063
+INFO 2025-05-19 15:13:26 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.27 loss:0.002 grdn:0.151 lr:7.6e-07 updt_s:0.222 data_s:0.067
+INFO 2025-05-19 15:14:24 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.32 loss:0.003 grdn:0.151 lr:7.4e-07 updt_s:0.222 data_s:0.068
+INFO 2025-05-19 15:15:23 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.37 loss:0.003 grdn:0.158 lr:7.2e-07 updt_s:0.222 data_s:0.069
+INFO 2025-05-19 15:16:21 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.42 loss:0.002 grdn:0.136 lr:7.1e-07 updt_s:0.222 data_s:0.072
+INFO 2025-05-19 15:17:20 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.48 loss:0.002 grdn:0.145 lr:6.9e-07 updt_s:0.222 data_s:0.072
+INFO 2025-05-19 15:18:19 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.53 loss:0.003 grdn:0.149 lr:6.7e-07 updt_s:0.222 data_s:0.069
+INFO 2025-05-19 15:19:18 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.58 loss:0.003 grdn:0.154 lr:6.6e-07 updt_s:0.222 data_s:0.075
+INFO 2025-05-19 15:20:18 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.63 loss:0.003 grdn:0.154 lr:6.4e-07 updt_s:0.221 data_s:0.080
+INFO 2025-05-19 15:21:27 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.68 loss:0.002 grdn:0.135 lr:6.3e-07 updt_s:0.230 data_s:0.114
+INFO 2025-05-19 15:22:33 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.73 loss:0.002 grdn:0.139 lr:6.1e-07 updt_s:0.221 data_s:0.107
+INFO 2025-05-19 15:23:40 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.78 loss:0.003 grdn:0.141 lr:5.9e-07 updt_s:0.221 data_s:0.112
+INFO 2025-05-19 15:24:46 ts/train.py:232 step:305K smpl:5M ep:23K epch:76.83 loss:0.002 grdn:0.127 lr:5.8e-07 updt_s:0.221 data_s:0.110
+INFO 2025-05-19 15:25:51 ts/train.py:232 step:305K smpl:5M ep:23K epch:76.88 loss:0.002 grdn:0.140 lr:5.6e-07 updt_s:0.221 data_s:0.103
+INFO 2025-05-19 15:26:56 ts/train.py:232 step:305K smpl:5M ep:23K epch:76.93 loss:0.002 grdn:0.136 lr:5.5e-07 updt_s:0.221 data_s:0.103
+INFO 2025-05-19 15:28:00 ts/train.py:232 step:305K smpl:5M ep:23K epch:76.98 loss:0.003 grdn:0.151 lr:5.4e-07 updt_s:0.221 data_s:0.098
+INFO 2025-05-19 15:29:03 ts/train.py:232 step:305K smpl:5M ep:23K epch:77.03 loss:0.002 grdn:0.143 lr:5.2e-07 updt_s:0.221 data_s:0.090
+INFO 2025-05-19 15:30:05 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.08 loss:0.003 grdn:0.149 lr:5.1e-07 updt_s:0.221 data_s:0.090
+INFO 2025-05-19 15:31:07 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.13 loss:0.003 grdn:0.154 lr:4.9e-07 updt_s:0.221 data_s:0.088
+INFO 2025-05-19 15:32:10 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.18 loss:0.003 grdn:0.157 lr:4.8e-07 updt_s:0.221 data_s:0.091
+INFO 2025-05-19 15:33:13 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.23 loss:0.002 grdn:0.132 lr:4.7e-07 updt_s:0.222 data_s:0.091
+INFO 2025-05-19 15:34:14 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.28 loss:0.002 grdn:0.136 lr:4.5e-07 updt_s:0.221 data_s:0.086
+INFO 2025-05-19 15:35:17 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.33 loss:0.002 grdn:0.134 lr:4.4e-07 updt_s:0.221 data_s:0.093
+INFO 2025-05-19 15:36:20 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.38 loss:0.003 grdn:0.158 lr:4.3e-07 updt_s:0.221 data_s:0.090
+INFO 2025-05-19 15:37:23 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.43 loss:0.003 grdn:0.149 lr:4.1e-07 updt_s:0.221 data_s:0.093
+INFO 2025-05-19 15:38:25 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.48 loss:0.002 grdn:0.144 lr:4.0e-07 updt_s:0.221 data_s:0.089
+INFO 2025-05-19 15:39:28 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.53 loss:0.002 grdn:0.145 lr:3.9e-07 updt_s:0.221 data_s:0.092
+INFO 2025-05-19 15:40:30 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.58 loss:0.002 grdn:0.142 lr:3.8e-07 updt_s:0.221 data_s:0.088
+INFO 2025-05-19 15:41:38 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.64 loss:0.002 grdn:0.145 lr:3.7e-07 updt_s:0.221 data_s:0.117
+INFO 2025-05-19 15:42:45 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.69 loss:0.002 grdn:0.138 lr:3.5e-07 updt_s:0.221 data_s:0.116
+INFO 2025-05-19 15:43:53 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.74 loss:0.003 grdn:0.149 lr:3.4e-07 updt_s:0.221 data_s:0.115
+INFO 2025-05-19 15:45:00 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.79 loss:0.003 grdn:0.145 lr:3.3e-07 updt_s:0.221 data_s:0.115
+INFO 2025-05-19 15:46:08 ts/train.py:232 step:309K smpl:5M ep:23K epch:77.84 loss:0.002 grdn:0.132 lr:3.2e-07 updt_s:0.221 data_s:0.116
+INFO 2025-05-19 15:47:15 ts/train.py:232 step:309K smpl:5M ep:23K epch:77.89 loss:0.002 grdn:0.146 lr:3.1e-07 updt_s:0.221 data_s:0.115
+INFO 2025-05-19 15:48:20 ts/train.py:232 step:309K smpl:5M ep:23K epch:77.94 loss:0.003 grdn:0.150 lr:3.0e-07 updt_s:0.221 data_s:0.100
+INFO 2025-05-19 15:49:23 ts/train.py:232 step:309K smpl:5M ep:23K epch:77.99 loss:0.003 grdn:0.153 lr:2.9e-07 updt_s:0.221 data_s:0.095
+INFO 2025-05-19 15:50:26 ts/train.py:232 step:309K smpl:5M ep:23K epch:78.04 loss:0.003 grdn:0.146 lr:2.8e-07 updt_s:0.221 data_s:0.092
+INFO 2025-05-19 15:51:29 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.09 loss:0.002 grdn:0.149 lr:2.7e-07 updt_s:0.221 data_s:0.092
+INFO 2025-05-19 15:52:32 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.14 loss:0.002 grdn:0.139 lr:2.6e-07 updt_s:0.221 data_s:0.092
+INFO 2025-05-19 15:53:33 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.19 loss:0.002 grdn:0.137 lr:2.5e-07 updt_s:0.221 data_s:0.085
+INFO 2025-05-19 15:54:36 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.24 loss:0.003 grdn:0.150 lr:2.4e-07 updt_s:0.221 data_s:0.091
+INFO 2025-05-19 15:55:39 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.29 loss:0.003 grdn:0.155 lr:2.3e-07 updt_s:0.221 data_s:0.093
+INFO 2025-05-19 15:56:41 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.34 loss:0.003 grdn:0.143 lr:2.2e-07 updt_s:0.221 data_s:0.089
+INFO 2025-05-19 15:57:43 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.39 loss:0.003 grdn:0.156 lr:2.1e-07 updt_s:0.221 data_s:0.091
+INFO 2025-05-19 15:58:46 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.44 loss:0.002 grdn:0.146 lr:2.0e-07 updt_s:0.221 data_s:0.092
+INFO 2025-05-19 15:59:48 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.49 loss:0.002 grdn:0.140 lr:1.9e-07 updt_s:0.221 data_s:0.086
+INFO 2025-05-19 16:00:50 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.54 loss:0.002 grdn:0.134 lr:1.8e-07 updt_s:0.221 data_s:0.091
+INFO 2025-05-19 16:01:58 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.59 loss:0.002 grdn:0.142 lr:1.7e-07 updt_s:0.221 data_s:0.116
+INFO 2025-05-19 16:03:05 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.64 loss:0.003 grdn:0.161 lr:1.7e-07 updt_s:0.221 data_s:0.113
+INFO 2025-05-19 16:04:12 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.69 loss:0.002 grdn:0.138 lr:1.6e-07 updt_s:0.222 data_s:0.111
+INFO 2025-05-19 16:05:18 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.75 loss:0.002 grdn:0.131 lr:1.5e-07 updt_s:0.221 data_s:0.112
+INFO 2025-05-19 16:06:26 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.80 loss:0.002 grdn:0.135 lr:1.4e-07 updt_s:0.221 data_s:0.115
+INFO 2025-05-19 16:07:33 ts/train.py:232 step:313K smpl:5M ep:24K epch:78.85 loss:0.002 grdn:0.145 lr:1.4e-07 updt_s:0.221 data_s:0.111
+INFO 2025-05-19 16:08:39 ts/train.py:232 step:313K smpl:5M ep:24K epch:78.90 loss:0.003 grdn:0.148 lr:1.3e-07 updt_s:0.221 data_s:0.109
+INFO 2025-05-19 16:09:42 ts/train.py:232 step:313K smpl:5M ep:24K epch:78.95 loss:0.002 grdn:0.138 lr:1.2e-07 updt_s:0.222 data_s:0.095
+INFO 2025-05-19 16:10:45 ts/train.py:232 step:313K smpl:5M ep:24K epch:79.00 loss:0.002 grdn:0.140 lr:1.2e-07 updt_s:0.221 data_s:0.091
+INFO 2025-05-19 16:11:48 ts/train.py:232 step:313K smpl:5M ep:24K epch:79.05 loss:0.002 grdn:0.150 lr:1.1e-07 updt_s:0.221 data_s:0.094
+INFO 2025-05-19 16:12:52 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.10 loss:0.003 grdn:0.149 lr:1.0e-07 updt_s:0.221 data_s:0.095
+INFO 2025-05-19 16:13:56 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.15 loss:0.002 grdn:0.155 lr:9.6e-08 updt_s:0.221 data_s:0.098
+INFO 2025-05-19 16:15:01 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.20 loss:0.002 grdn:0.142 lr:9.0e-08 updt_s:0.223 data_s:0.101
+INFO 2025-05-19 16:16:04 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.25 loss:0.002 grdn:0.132 lr:8.4e-08 updt_s:0.221 data_s:0.094
+INFO 2025-05-19 16:17:08 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.30 loss:0.002 grdn:0.136 lr:7.9e-08 updt_s:0.221 data_s:0.099
+INFO 2025-05-19 16:18:12 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.35 loss:0.002 grdn:0.148 lr:7.3e-08 updt_s:0.221 data_s:0.096
+INFO 2025-05-19 16:19:16 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.40 loss:0.002 grdn:0.143 lr:6.8e-08 updt_s:0.221 data_s:0.097
+INFO 2025-05-19 16:20:19 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.45 loss:0.002 grdn:0.140 lr:6.3e-08 updt_s:0.221 data_s:0.096
+INFO 2025-05-19 16:21:23 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.50 loss:0.002 grdn:0.141 lr:5.8e-08 updt_s:0.221 data_s:0.096
+INFO 2025-05-19 16:22:28 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.55 loss:0.002 grdn:0.145 lr:5.3e-08 updt_s:0.221 data_s:0.107
+INFO 2025-05-19 16:23:37 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.60 loss:0.003 grdn:0.146 lr:4.9e-08 updt_s:0.222 data_s:0.120
+INFO 2025-05-19 16:24:45 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.65 loss:0.002 grdn:0.139 lr:4.5e-08 updt_s:0.221 data_s:0.117
+INFO 2025-05-19 16:25:52 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.70 loss:0.002 grdn:0.129 lr:4.1e-08 updt_s:0.221 data_s:0.113
+INFO 2025-05-19 16:26:59 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.75 loss:0.002 grdn:0.129 lr:3.7e-08 updt_s:0.221 data_s:0.115
+INFO 2025-05-19 16:28:06 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.80 loss:0.002 grdn:0.136 lr:3.3e-08 updt_s:0.221 data_s:0.111
+INFO 2025-05-19 16:29:11 ts/train.py:232 step:317K smpl:5M ep:24K epch:79.85 loss:0.003 grdn:0.154 lr:3.0e-08 updt_s:0.222 data_s:0.104
+INFO 2025-05-19 16:30:14 ts/train.py:232 step:317K smpl:5M ep:24K epch:79.91 loss:0.002 grdn:0.133 lr:2.6e-08 updt_s:0.221 data_s:0.091
+INFO 2025-05-19 16:31:15 ts/train.py:232 step:317K smpl:5M ep:24K epch:79.96 loss:0.003 grdn:0.149 lr:2.3e-08 updt_s:0.221 data_s:0.086
+INFO 2025-05-19 16:32:18 ts/train.py:232 step:317K smpl:5M ep:24K epch:80.01 loss:0.002 grdn:0.141 lr:2.0e-08 updt_s:0.221 data_s:0.089
+INFO 2025-05-19 16:33:19 ts/train.py:232 step:317K smpl:5M ep:24K epch:80.06 loss:0.002 grdn:0.148 lr:1.8e-08 updt_s:0.222 data_s:0.083
+INFO 2025-05-19 16:34:21 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.11 loss:0.002 grdn:0.138 lr:1.5e-08 updt_s:0.222 data_s:0.087
+INFO 2025-05-19 16:35:22 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.16 loss:0.003 grdn:0.152 lr:1.3e-08 updt_s:0.221 data_s:0.084
+INFO 2025-05-19 16:36:25 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.21 loss:0.002 grdn:0.130 lr:1.1e-08 updt_s:0.221 data_s:0.094
+INFO 2025-05-19 16:37:27 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.26 loss:0.002 grdn:0.140 lr:8.7e-09 updt_s:0.221 data_s:0.089
+INFO 2025-05-19 16:38:30 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.31 loss:0.002 grdn:0.140 lr:7.0e-09 updt_s:0.222 data_s:0.091
+INFO 2025-05-19 16:39:32 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.36 loss:0.002 grdn:0.140 lr:5.4e-09 updt_s:0.222 data_s:0.088
+INFO 2025-05-19 16:40:35 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.41 loss:0.002 grdn:0.132 lr:4.1e-09 updt_s:0.221 data_s:0.092
+INFO 2025-05-19 16:41:38 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.46 loss:0.002 grdn:0.146 lr:2.9e-09 updt_s:0.221 data_s:0.090
+INFO 2025-05-19 16:42:42 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.51 loss:0.002 grdn:0.143 lr:2.0e-09 updt_s:0.221 data_s:0.097
+INFO 2025-05-19 16:43:49 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.56 loss:0.002 grdn:0.136 lr:1.2e-09 updt_s:0.221 data_s:0.113
+INFO 2025-05-19 16:44:56 ts/train.py:232 step:320K smpl:5M ep:24K epch:80.61 loss:0.002 grdn:0.148 lr:6.1e-10 updt_s:0.221 data_s:0.113
+INFO 2025-05-19 16:46:03 ts/train.py:232 step:320K smpl:5M ep:24K epch:80.66 loss:0.003 grdn:0.157 lr:2.2e-10 updt_s:0.221 data_s:0.114
+INFO 2025-05-19 16:47:08 ts/train.py:232 step:320K smpl:5M ep:24K epch:80.71 loss:0.002 grdn:0.146 lr:3.2e-11 updt_s:0.222 data_s:0.104
+INFO 2025-05-19 16:47:08 ts/train.py:241 Checkpoint policy after step 320000
+INFO 2025-05-19 16:47:14 ts/train.py:283 End of training

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/files/requirements.txt

@@ -0,0 +1,146 @@
+pyyaml-include==1.4.1
+pillow==11.2.1
+six==1.17.0
+nvidia-cusparse-cu12==12.3.1.170
+packaging==24.2
+torch==2.6.0
+async-timeout==5.0.1
+wcwidth==0.2.13
+nvidia-nvjitlink-cu12==12.4.127
+pip==25.0.1
+requests==2.32.3
+draccus==0.10.0
+python-dateutil==2.9.0.post0
+numba==0.61.2
+antlr4-python3-runtime==4.9.3
+pyzmq==26.4.0
+termcolor==3.0.1
+pyserial==3.5
+pynput==1.8.1
+pfzy==0.3.4
+deepdiff==8.4.2
+h5py==3.13.0
+imageio==2.37.0
+typing-inspect==0.9.0
+imageio-ffmpeg==0.6.0
+multidict==6.4.3
+fasteners==0.19
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cusolver-cu12==11.6.1.9
+triton==3.2.0
+beautifulsoup4==4.13.4
+annotated-types==0.7.0
+llvmlite==0.44.0
+TorchCodec==0.2.1
+hf_transfer==0.1.9
+cffi==1.17.1
+zarr==2.18.3
+opencv-python-headless==4.11.0.86
+tqdm==4.67.1
+protobuf==5.29.4
+av==14.3.0
+click==8.1.8
+pyarrow==19.0.1
+Flask==3.1.0
+zipp==3.21.0
+importlib_metadata==8.6.1
+urllib3==2.4.0
+huggingface-hub==0.30.2
+gitdb==4.0.12
+nvidia-nccl-cu12==2.21.5
+pydantic_core==2.33.1
+MarkupSafe==3.0.2
+blinker==1.9.0
+aiohappyeyeballs==2.6.1
+einops==0.8.1
+feetech-servo-sdk==1.0.0
+omegaconf==2.3.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+gdown==5.2.0
+pycparser==2.22
+pandas==2.2.3
+orderly-set==5.4.0
+certifi==2025.1.31
+torchvision==0.21.0
+sentry-sdk==2.26.1
+dill==0.3.8
+mpmath==1.3.0
+python-xlib==0.33
+mergedeep==1.3.4
+Werkzeug==3.1.3
+prompt_toolkit==3.0.51
+PyYAML==6.0.2
+multiprocess==0.70.16
+toml==0.10.2
+wheel==0.45.1
+platformdirs==4.3.7
+numcodecs==0.13.1
+evdev==1.9.1
+pydantic==2.11.3
+fsspec==2024.12.0
+xxhash==3.5.0
+setproctitle==1.3.5
+propcache==0.3.1
+nvidia-cuda-cupti-cu12==12.4.127
+GitPython==3.1.44
+cloudpickle==3.1.1
+frozenlist==1.6.0
+filelock==3.18.0
+idna==3.10
+lerobot==0.1.0
+Farama-Notifications==0.0.4
+charset-normalizer==3.4.1
+wandb==0.19.9
+tzdata==2025.2
+nvidia-cublas-cu12==12.4.5.8
+pytz==2025.2
+smmap==5.0.2
+networkx==3.4.2
+numpy==2.2.4
+gymnasium==0.29.1
+aiohttp==3.11.16
+psutil==7.0.0
+attrs==25.3.0
+inquirerpy==0.3.4
+nvidia-curand-cu12==10.3.5.147
+nvidia-cudnn-cu12==9.1.0.70
+itsdangerous==2.2.0
+asciitree==0.3.3
+rerun-sdk==0.22.1
+mypy-extensions==1.0.0
+regex==2024.11.6
+aiosignal==1.3.2
+docker-pycreds==0.4.0
+sympy==1.13.1
+PySocks==1.7.1
+yarl==1.20.0
+nvidia-cuda-runtime-cu12==12.4.127
+soupsieve==2.6
+Jinja2==3.1.6
+nvidia-nvtx-cu12==12.4.127
+safetensors==0.5.3
+datasets==3.5.0
+typing_extensions==4.13.2
+diffusers==0.33.1
+pymunk==6.11.1
+nvidia-cusparselt-cu12==0.6.2
+typing-inspection==0.4.0
+setuptools==78.1.0
+cmake==4.0.0
+jsonlines==4.0.0
+zipp==3.19.2
+packaging==24.2
+importlib_metadata==8.0.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+backports.tarfile==1.2.0
+autocommand==2.2.2
+inflect==7.3.1
+typeguard==4.3.0
+more-itertools==10.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+tomli==2.0.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+jaraco.text==3.12.1

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/files/wandb-metadata.json

@@ -0,0 +1,90 @@
+{
+  "os": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.17",
+  "startedAt": "2025-05-19T15:01:39.148320Z",
+  "args": [
+    "--resume=true",
+    "--config_path=/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/checkpoints/300000/pretrained_model/train_config.json",
+    "--save_freq=20000",
+    "--steps=320000",
+    "--wandb.run_id=5e56tibb"
+  ],
+  "program": "/NAS/LSY/lerobot/lerobot/scripts/train.py",
+  "codePath": "lerobot/scripts/train.py",
+  "git": {
+    "remote": "https://github.com/huggingface/lerobot.git",
+    "commit": "a445d9c9da6bea99a8972daa4fe1fdd053d711d2"
+  },
+  "root": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain",
+  "host": "91cf0fc7c553",
+  "executable": "/opt/conda/envs/lerobot/bin/python",
+  "codePathLocal": "lerobot/scripts/train.py",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA GeForce RTX 3090",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1820335460352",
+      "used": "1158311636992"
+    }
+  },
+  "memory": {
+    "total": "253372637184"
+  },
+  "cpu": {
+    "count": 48,
+    "countLogical": 96
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.2"
+}

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/steps":320000,"train/lr":3.1986937902489527e-11,"train/loss":0.0023125900846207516,"_step":320000,"train/grad_norm":0.1458555322047323,"train/epochs":80.71254039568062,"train/samples":5120000,"train/episodes":24213.762118704188,"_runtime":6336.158246907,"train/update_s":0.2218399324489292,"_wandb":{"runtime":6336},"train/dataloading_s":0.1041360501863528,"_timestamp":1.7476732288169024e+09}

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-05-19T15:01:38.586929501Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp1u1y4t1j/port-827647.txt","pid":827647,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-05-19T15:01:38.591906772Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":39277,"Zone":""}}
+{"time":"2025-05-19T15:01:38.592160783Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":827647}
+{"time":"2025-05-19T15:01:38.76709357Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58104"}
+{"time":"2025-05-19T15:01:38.775741601Z","level":"INFO","msg":"handleInformInit: received","streamId":"5e56tibb","id":"127.0.0.1:58104"}
+{"time":"2025-05-19T15:01:39.112943232Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"5e56tibb","id":"127.0.0.1:58104"}
+{"time":"2025-05-19T16:47:15.306413537Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:58104"}
+{"time":"2025-05-19T16:47:15.306537015Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-05-19T16:47:15.306525701Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:58104"}
+{"time":"2025-05-19T16:47:15.306709243Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:58104"}
+{"time":"2025-05-19T16:47:34.715577816Z","level":"ERROR","msg":"processOutgoingData: flush error","error":"write tcp 127.0.0.1:39277->127.0.0.1:58104: use of closed network connection","id":"127.0.0.1:58104"}
+{"time":"2025-05-19T16:47:35.025804238Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:58104"}
+{"time":"2025-05-19T16:47:35.025864301Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:58104"}
+{"time":"2025-05-19T16:47:35.025887488Z","level":"INFO","msg":"server is closed"}

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/logs/debug-internal.log

@@ -0,0 +1,393 @@
+{"time":"2025-05-19T15:01:38.775891619Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/logs/debug-core.log"}
+{"time":"2025-05-19T15:01:39.112865708Z","level":"INFO","msg":"created new stream","id":"5e56tibb"}
+{"time":"2025-05-19T15:01:39.112931498Z","level":"INFO","msg":"stream: started","id":"5e56tibb"}
+{"time":"2025-05-19T15:01:39.112972146Z","level":"INFO","msg":"sender: started","stream_id":"5e56tibb"}
+{"time":"2025-05-19T15:01:39.113088081Z","level":"INFO","msg":"handler: started","stream_id":"5e56tibb"}
+{"time":"2025-05-19T15:01:39.113144792Z","level":"INFO","msg":"writer: Do: started","stream_id":"5e56tibb"}
+{"time":"2025-05-19T15:01:40.811234696Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-05-19T15:02:48.970606405Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970718569Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970728976Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970738404Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970746925Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970754467Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970762569Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970769204Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:02:48.970778213Z","level":"WARN","msg":"handler: ignoring partial history record","step":300200,"current":308401}
+{"time":"2025-05-19T15:03:49.51029375Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510388803Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510402073Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510413876Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510424841Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510437063Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510447469Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510458365Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:03:49.510468771Z","level":"WARN","msg":"handler: ignoring partial history record","step":300400,"current":308401}
+{"time":"2025-05-19T15:04:49.410214863Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410272062Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.41028205Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410291548Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410300488Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410308589Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410330729Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.410339249Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:04:49.41034784Z","level":"WARN","msg":"handler: ignoring partial history record","step":300600,"current":308401}
+{"time":"2025-05-19T15:05:45.346295957Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346378718Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346388635Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346396667Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346404699Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346411334Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346426978Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.346435499Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:05:45.3464436Z","level":"WARN","msg":"handler: ignoring partial history record","step":300800,"current":308401}
+{"time":"2025-05-19T15:06:43.322171399Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322258839Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322268338Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322276858Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.32228489Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322293899Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322300953Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.322308566Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:06:43.32231562Z","level":"WARN","msg":"handler: ignoring partial history record","step":301000,"current":308401}
+{"time":"2025-05-19T15:07:40.490220028Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490282047Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490292872Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490301882Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490309913Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490318434Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490326466Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490334497Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:07:40.490342599Z","level":"WARN","msg":"handler: ignoring partial history record","step":301200,"current":308401}
+{"time":"2025-05-19T15:08:38.494324851Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494451681Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494463973Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494474869Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494484297Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494493726Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494504621Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494513211Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:08:38.494523617Z","level":"WARN","msg":"handler: ignoring partial history record","step":301400,"current":308401}
+{"time":"2025-05-19T15:09:35.618275757Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618372696Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618387363Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618399236Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618411458Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618424797Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618436531Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618448404Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:09:35.618460207Z","level":"WARN","msg":"handler: ignoring partial history record","step":301600,"current":308401}
+{"time":"2025-05-19T15:10:32.982355474Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982486984Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982499276Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982508705Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982518203Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982527632Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.98253713Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982546628Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:10:32.982555149Z","level":"WARN","msg":"handler: ignoring partial history record","step":301800,"current":308401}
+{"time":"2025-05-19T15:11:31.670364782Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670516127Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670528419Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670564876Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670575282Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670586178Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670597562Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670606501Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:11:31.670615511Z","level":"WARN","msg":"handler: ignoring partial history record","step":302000,"current":308401}
+{"time":"2025-05-19T15:12:28.781744515Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.78179934Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.781809258Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.781895441Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782026463Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782112087Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782210493Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782306594Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:12:28.782390333Z","level":"WARN","msg":"handler: ignoring partial history record","step":302200,"current":308401}
+{"time":"2025-05-19T15:13:26.696961702Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.69703273Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697092305Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697202513Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697300919Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697417343Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697481178Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697631615Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:13:26.697674707Z","level":"WARN","msg":"handler: ignoring partial history record","step":302400,"current":308401}
+{"time":"2025-05-19T15:14:24.710189485Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710262818Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710272805Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710282304Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710290754Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710298367Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710317224Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710325256Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:14:24.710333287Z","level":"WARN","msg":"handler: ignoring partial history record","step":302600,"current":308401}
+{"time":"2025-05-19T15:15:23.066204083Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066264635Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066279721Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066294457Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066307168Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066320019Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.06633273Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066345511Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:15:23.066357314Z","level":"WARN","msg":"handler: ignoring partial history record","step":302800,"current":308401}
+{"time":"2025-05-19T15:16:22.030202312Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030301207Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030311124Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030321042Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030329562Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030338572Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030346673Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.030356102Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:16:22.03036553Z","level":"WARN","msg":"handler: ignoring partial history record","step":303000,"current":308401}
+{"time":"2025-05-19T15:17:20.850184668Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850264146Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850272667Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850281188Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850289219Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850296343Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850303886Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850311498Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:17:20.850318064Z","level":"WARN","msg":"handler: ignoring partial history record","step":303200,"current":308401}
+{"time":"2025-05-19T15:18:19.162114604Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.16216894Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162180324Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162189333Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162198762Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162208679Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162216711Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162226698Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:18:19.162236127Z","level":"WARN","msg":"handler: ignoring partial history record","step":303400,"current":308401}
+{"time":"2025-05-19T15:19:18.598200374Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598296963Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598307789Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598316798Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.59832483Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598334328Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.59834236Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.598351369Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:19:18.59835982Z","level":"WARN","msg":"handler: ignoring partial history record","step":303600,"current":308401}
+{"time":"2025-05-19T15:20:18.890268849Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.89034742Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890356848Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.89036495Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890372982Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890383318Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.89039135Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890400848Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:20:18.890409369Z","level":"WARN","msg":"handler: ignoring partial history record","step":303800,"current":308401}
+{"time":"2025-05-19T15:21:27.978855062Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.978924554Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979014928Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979107676Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979200914Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979292684Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979380195Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979484746Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:21:27.979581755Z","level":"WARN","msg":"handler: ignoring partial history record","step":304000,"current":308401}
+{"time":"2025-05-19T15:22:33.850208001Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850308782Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850320585Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850332458Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850340978Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850349499Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850358508Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850366959Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:22:33.850374502Z","level":"WARN","msg":"handler: ignoring partial history record","step":304200,"current":308401}
+{"time":"2025-05-19T15:23:40.554189834Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554286843Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554295852Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.55430577Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554314221Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554322252Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554330354Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554339293Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:23:40.554347814Z","level":"WARN","msg":"handler: ignoring partial history record","step":304400,"current":308401}
+{"time":"2025-05-19T15:24:46.958238203Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958303993Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958315796Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958326622Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.95833619Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958345618Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958368316Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958376348Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:24:46.958385846Z","level":"WARN","msg":"handler: ignoring partial history record","step":304600,"current":308401}
+{"time":"2025-05-19T15:25:51.910176789Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.910250122Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.91026004Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.910269538Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.9102775Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.91028602Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.91029503Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.91030355Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:25:51.910311582Z","level":"WARN","msg":"handler: ignoring partial history record","step":304800,"current":308401}
+{"time":"2025-05-19T15:26:56.877751832Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.877905132Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.877916936Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.877927272Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878013944Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878177092Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878222978Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878298266Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:26:56.878370202Z","level":"WARN","msg":"handler: ignoring partial history record","step":305000,"current":308401}
+{"time":"2025-05-19T15:28:00.946443925Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.946623695Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.946669161Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.946785516Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.946877287Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.947022625Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.947118098Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.947227887Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:28:00.947368896Z","level":"WARN","msg":"handler: ignoring partial history record","step":305200,"current":308401}
+{"time":"2025-05-19T15:29:03.382273603Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382338416Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382349241Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.38235825Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382366771Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382376688Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382386117Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382395196Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:29:03.382405114Z","level":"WARN","msg":"handler: ignoring partial history record","step":305400,"current":308401}
+{"time":"2025-05-19T15:30:05.886211776Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886300683Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886311579Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886320588Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886329039Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886453076Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886544358Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886555672Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:30:05.886566637Z","level":"WARN","msg":"handler: ignoring partial history record","step":305600,"current":308401}
+{"time":"2025-05-19T15:31:07.938290568Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938380942Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938392326Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938401335Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938410345Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938418865Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938426827Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938435348Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:31:07.938443868Z","level":"WARN","msg":"handler: ignoring partial history record","step":305800,"current":308401}
+{"time":"2025-05-19T15:32:10.554145176Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554221302Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554244978Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554253988Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554262928Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554271448Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554280946Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554290375Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:32:10.554299384Z","level":"WARN","msg":"handler: ignoring partial history record","step":306000,"current":308401}
+{"time":"2025-05-19T15:33:13.330208354Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330282175Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330292163Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330301661Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330310112Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.330319121Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.33032855Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.33033707Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:33:13.33034608Z","level":"WARN","msg":"handler: ignoring partial history record","step":306200,"current":308401}
+{"time":"2025-05-19T15:34:15.038319398Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038438198Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038450909Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038460896Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038471302Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038482616Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038493581Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038504407Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:34:15.038514813Z","level":"WARN","msg":"handler: ignoring partial history record","step":306400,"current":308401}
+{"time":"2025-05-19T15:35:18.042214537Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042282702Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042294505Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042303933Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042313432Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042341368Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042350797Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042360295Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:35:18.042368815Z","level":"WARN","msg":"handler: ignoring partial history record","step":306600,"current":308401}
+{"time":"2025-05-19T15:36:20.478114919Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478200474Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478211858Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478221287Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478231274Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478241191Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478251109Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478260118Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:36:20.478269547Z","level":"WARN","msg":"handler: ignoring partial history record","step":306800,"current":308401}
+{"time":"2025-05-19T15:37:23.530156874Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530242499Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530253813Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530263731Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530271832Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530279864Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530287895Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530295927Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:37:23.530303959Z","level":"WARN","msg":"handler: ignoring partial history record","step":307000,"current":308401}
+{"time":"2025-05-19T15:38:25.694242948Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694304059Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694313976Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694323894Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694333811Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.69434324Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694352249Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.694362236Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:38:25.69438689Z","level":"WARN","msg":"handler: ignoring partial history record","step":307200,"current":308401}
+{"time":"2025-05-19T15:39:28.571758776Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.571865214Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.571936172Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572038838Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572129701Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572211554Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572297109Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572391813Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:39:28.572472689Z","level":"WARN","msg":"handler: ignoring partial history record","step":307400,"current":308401}
+{"time":"2025-05-19T15:40:30.590914294Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.590993773Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591096439Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591179759Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591271041Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591374615Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591458843Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591561997Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:40:30.591641895Z","level":"WARN","msg":"handler: ignoring partial history record","step":307600,"current":308401}
+{"time":"2025-05-19T15:41:38.278181827Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278239516Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278249922Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278258932Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278267452Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278276462Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278284912Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278292944Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:41:38.278300557Z","level":"WARN","msg":"handler: ignoring partial history record","step":307800,"current":308401}
+{"time":"2025-05-19T15:42:45.914075361Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914133119Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914156795Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914165316Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914173836Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914182776Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914191297Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914198839Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:42:45.914206941Z","level":"WARN","msg":"handler: ignoring partial history record","step":308000,"current":308401}
+{"time":"2025-05-19T15:43:53.270063574Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270127479Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270138793Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270146824Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270154856Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270163377Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270171408Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.27017951Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:43:53.270188031Z","level":"WARN","msg":"handler: ignoring partial history record","step":308200,"current":308401}
+{"time":"2025-05-19T15:45:00.713439117Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.713600938Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.713724417Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.713856416Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.713973678Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.714098623Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.714170071Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.714298717Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T15:45:00.714359828Z","level":"WARN","msg":"handler: ignoring partial history record","step":308400,"current":308401}
+{"time":"2025-05-19T16:47:15.306554546Z","level":"INFO","msg":"stream: closing","id":"5e56tibb"}
+{"time":"2025-05-19T16:47:15.306584367Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-05-19T16:47:15.306645827Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-05-19T16:47:34.765612049Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-05-19T16:47:35.023526244Z","level":"INFO","msg":"handler: closed","stream_id":"5e56tibb"}
+{"time":"2025-05-19T16:47:35.023574923Z","level":"INFO","msg":"writer: Close: closed","stream_id":"5e56tibb"}
+{"time":"2025-05-19T16:47:35.023585399Z","level":"INFO","msg":"sender: closed","stream_id":"5e56tibb"}
+{"time":"2025-05-19T16:47:35.02365964Z","level":"INFO","msg":"stream: closed","id":"5e56tibb"}

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/logs/debug.log

@@ -0,0 +1,24 @@
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Configure stats pid to 827647
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/logs/debug.log
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/logs/debug-internal.log
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:init():781] calling init triggers
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'shylee/so100_pengripG', 'root': '/SSD/LSY/lerobot_dataset/pengripG', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MIN_MAX: 'MIN_MAX'>, 'ACTION': <NormalizationMode.MIN_MAX: 'MIN_MAX'>}, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [6]}, 'observation.images.FrontCam': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 480, 640]}, 'observation.images.TopCam': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 480, 640]}, 'observation.images.WristCam': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 480, 640]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ac': False, 'ac_threshold': 0.99, 'use_pcg': False, 'pcg_weight': 0.1, 'use_bid': False, 'bid_batch_size': 15, 'bid_mode_size': 3, 'bid_temporal_coherence': 0.5, 'weak_model_path': None, 'weak_model_config': None}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain', 'job_name': 'DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain', 'resume': True, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 320000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '5e56tibb', 'mode': None}, '_wandb': {}}
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:init():809] starting backend
+2025-05-19 15:01:38,769 INFO    MainThread:827647 [wandb_init.py:init():813] sending inform_init request
+2025-05-19 15:01:38,773 INFO    MainThread:827647 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-05-19 15:01:38,774 INFO    MainThread:827647 [wandb_init.py:init():823] backend started and connected
+2025-05-19 15:01:39,150 INFO    MainThread:827647 [wandb_init.py:init():915] updated telemetry
+2025-05-19 15:01:39,842 INFO    MainThread:827647 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
+2025-05-19 15:01:40,769 INFO    MainThread:827647 [wandb_init.py:init():1009] run resumed
+2025-05-19 15:01:40,772 INFO    MainThread:827647 [wandb_init.py:init():1014] starting run threads in backend
+2025-05-19 15:01:40,956 INFO    MainThread:827647 [wandb_run.py:_console_start():2454] atexit reg
+2025-05-19 15:01:40,956 INFO    MainThread:827647 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-05-19 15:01:40,956 INFO    MainThread:827647 [wandb_run.py:_redirect():2371] Wrapping output streams.
+2025-05-19 15:01:40,957 INFO    MainThread:827647 [wandb_run.py:_redirect():2394] Redirects installed.
+2025-05-19 15:01:40,958 INFO    MainThread:827647 [wandb_init.py:init():1056] run started, returning control to user process
+2025-05-19 16:47:15,305 INFO    MsgRouterThr:827647 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.

DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/run-5e56tibb.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0e8a60ec2976ae55cb411a272cd7b73a90394c2ab0d0ff5a3d8b278ba714244
+size 1382661