zzz99's picture
Training in progress, epoch 1
ca0cb2e verified
raw
history blame
21.1 kB
{
"os": "Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35",
"python": "3.10.13",
"heartbeatAt": "2024-02-08T18:53:11.692683",
"startedAt": "2024-02-08T18:53:11.292051",
"docker": null,
"cuda": null,
"args": [],
"state": "running",
"program": "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py",
"codePathLocal": null,
"host": "default",
"username": "sagemaker-user",
"executable": "/opt/conda/bin/python3",
"cpu_count": 96,
"cpu_count_logical": 192,
"cpu_freq": {
"current": 3259.3440989583337,
"min": 0.0,
"max": 0.0
},
"cpu_freq_per_core": [
{
"current": 3299.457,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.806,
"min": 0.0,
"max": 0.0
},
{
"current": 3205.787,
"min": 0.0,
"max": 0.0
},
{
"current": 3178.002,
"min": 0.0,
"max": 0.0
},
{
"current": 3200.706,
"min": 0.0,
"max": 0.0
},
{
"current": 3196.209,
"min": 0.0,
"max": 0.0
},
{
"current": 3188.711,
"min": 0.0,
"max": 0.0
},
{
"current": 3189.097,
"min": 0.0,
"max": 0.0
},
{
"current": 3206.992,
"min": 0.0,
"max": 0.0
},
{
"current": 3200.764,
"min": 0.0,
"max": 0.0
},
{
"current": 2810.939,
"min": 0.0,
"max": 0.0
},
{
"current": 2847.942,
"min": 0.0,
"max": 0.0
},
{
"current": 2923.114,
"min": 0.0,
"max": 0.0
},
{
"current": 3028.23,
"min": 0.0,
"max": 0.0
},
{
"current": 3012.472,
"min": 0.0,
"max": 0.0
},
{
"current": 3044.914,
"min": 0.0,
"max": 0.0
},
{
"current": 2953.37,
"min": 0.0,
"max": 0.0
},
{
"current": 2957.586,
"min": 0.0,
"max": 0.0
},
{
"current": 2984.294,
"min": 0.0,
"max": 0.0
},
{
"current": 2961.352,
"min": 0.0,
"max": 0.0
},
{
"current": 2901.559,
"min": 0.0,
"max": 0.0
},
{
"current": 2801.726,
"min": 0.0,
"max": 0.0
},
{
"current": 2985.17,
"min": 0.0,
"max": 0.0
},
{
"current": 2963.11,
"min": 0.0,
"max": 0.0
},
{
"current": 2912.001,
"min": 0.0,
"max": 0.0
},
{
"current": 2965.712,
"min": 0.0,
"max": 0.0
},
{
"current": 2966.821,
"min": 0.0,
"max": 0.0
},
{
"current": 2871.172,
"min": 0.0,
"max": 0.0
},
{
"current": 2974.758,
"min": 0.0,
"max": 0.0
},
{
"current": 2989.099,
"min": 0.0,
"max": 0.0
},
{
"current": 2948.999,
"min": 0.0,
"max": 0.0
},
{
"current": 2895.266,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.988,
"min": 0.0,
"max": 0.0
},
{
"current": 2924.435,
"min": 0.0,
"max": 0.0
},
{
"current": 2919.839,
"min": 0.0,
"max": 0.0
},
{
"current": 2875.943,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.697,
"min": 0.0,
"max": 0.0
},
{
"current": 2805.016,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.583,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.604,
"min": 0.0,
"max": 0.0
},
{
"current": 2673.256,
"min": 0.0,
"max": 0.0
},
{
"current": 3296.503,
"min": 0.0,
"max": 0.0
},
{
"current": 3139.11,
"min": 0.0,
"max": 0.0
},
{
"current": 3137.942,
"min": 0.0,
"max": 0.0
},
{
"current": 2833.969,
"min": 0.0,
"max": 0.0
},
{
"current": 3153.277,
"min": 0.0,
"max": 0.0
},
{
"current": 3178.769,
"min": 0.0,
"max": 0.0
},
{
"current": 3207.604,
"min": 0.0,
"max": 0.0
},
{
"current": 2892.532,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.772,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.641,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.096,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.515,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.26,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.084,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.903,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.866,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.276,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.704,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.342,
"min": 0.0,
"max": 0.0
},
{
"current": 3297.795,
"min": 0.0,
"max": 0.0
},
{
"current": 3297.923,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.013,
"min": 0.0,
"max": 0.0
},
{
"current": 3297.43,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.564,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.54,
"min": 0.0,
"max": 0.0
},
{
"current": 2747.263,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.353,
"min": 0.0,
"max": 0.0
},
{
"current": 3297.896,
"min": 0.0,
"max": 0.0
},
{
"current": 2533.725,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.656,
"min": 0.0,
"max": 0.0
},
{
"current": 3293.031,
"min": 0.0,
"max": 0.0
},
{
"current": 3027.834,
"min": 0.0,
"max": 0.0
},
{
"current": 3024.556,
"min": 0.0,
"max": 0.0
},
{
"current": 3067.379,
"min": 0.0,
"max": 0.0
},
{
"current": 3010.826,
"min": 0.0,
"max": 0.0
},
{
"current": 3101.81,
"min": 0.0,
"max": 0.0
},
{
"current": 2973.599,
"min": 0.0,
"max": 0.0
},
{
"current": 3061.27,
"min": 0.0,
"max": 0.0
},
{
"current": 3291.322,
"min": 0.0,
"max": 0.0
},
{
"current": 3017.723,
"min": 0.0,
"max": 0.0
},
{
"current": 2660.496,
"min": 0.0,
"max": 0.0
},
{
"current": 3004.775,
"min": 0.0,
"max": 0.0
},
{
"current": 3021.086,
"min": 0.0,
"max": 0.0
},
{
"current": 3027.592,
"min": 0.0,
"max": 0.0
},
{
"current": 3059.589,
"min": 0.0,
"max": 0.0
},
{
"current": 3019.568,
"min": 0.0,
"max": 0.0
},
{
"current": 3029.623,
"min": 0.0,
"max": 0.0
},
{
"current": 3080.312,
"min": 0.0,
"max": 0.0
},
{
"current": 3066.263,
"min": 0.0,
"max": 0.0
},
{
"current": 2998.37,
"min": 0.0,
"max": 0.0
},
{
"current": 2949.133,
"min": 0.0,
"max": 0.0
},
{
"current": 2964.0,
"min": 0.0,
"max": 0.0
},
{
"current": 3222.788,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.63,
"min": 0.0,
"max": 0.0
},
{
"current": 2916.281,
"min": 0.0,
"max": 0.0
},
{
"current": 2825.282,
"min": 0.0,
"max": 0.0
},
{
"current": 3038.106,
"min": 0.0,
"max": 0.0
},
{
"current": 2895.235,
"min": 0.0,
"max": 0.0
},
{
"current": 3092.874,
"min": 0.0,
"max": 0.0
},
{
"current": 2924.994,
"min": 0.0,
"max": 0.0
},
{
"current": 2913.404,
"min": 0.0,
"max": 0.0
},
{
"current": 2935.638,
"min": 0.0,
"max": 0.0
},
{
"current": 2583.261,
"min": 0.0,
"max": 0.0
},
{
"current": 3101.162,
"min": 0.0,
"max": 0.0
},
{
"current": 3063.704,
"min": 0.0,
"max": 0.0
},
{
"current": 3093.23,
"min": 0.0,
"max": 0.0
},
{
"current": 3095.386,
"min": 0.0,
"max": 0.0
},
{
"current": 2925.773,
"min": 0.0,
"max": 0.0
},
{
"current": 2920.019,
"min": 0.0,
"max": 0.0
},
{
"current": 2916.15,
"min": 0.0,
"max": 0.0
},
{
"current": 2944.025,
"min": 0.0,
"max": 0.0
},
{
"current": 3259.667,
"min": 0.0,
"max": 0.0
},
{
"current": 3049.572,
"min": 0.0,
"max": 0.0
},
{
"current": 3263.675,
"min": 0.0,
"max": 0.0
},
{
"current": 3074.497,
"min": 0.0,
"max": 0.0
},
{
"current": 2923.985,
"min": 0.0,
"max": 0.0
},
{
"current": 2910.425,
"min": 0.0,
"max": 0.0
},
{
"current": 2812.861,
"min": 0.0,
"max": 0.0
},
{
"current": 2874.988,
"min": 0.0,
"max": 0.0
},
{
"current": 3120.953,
"min": 0.0,
"max": 0.0
},
{
"current": 3124.25,
"min": 0.0,
"max": 0.0
},
{
"current": 3113.753,
"min": 0.0,
"max": 0.0
},
{
"current": 3119.282,
"min": 0.0,
"max": 0.0
},
{
"current": 2982.281,
"min": 0.0,
"max": 0.0
},
{
"current": 3048.291,
"min": 0.0,
"max": 0.0
},
{
"current": 2987.986,
"min": 0.0,
"max": 0.0
},
{
"current": 2733.968,
"min": 0.0,
"max": 0.0
},
{
"current": 3274.202,
"min": 0.0,
"max": 0.0
},
{
"current": 3120.154,
"min": 0.0,
"max": 0.0
},
{
"current": 3122.388,
"min": 0.0,
"max": 0.0
},
{
"current": 2592.46,
"min": 0.0,
"max": 0.0
},
{
"current": 3121.448,
"min": 0.0,
"max": 0.0
},
{
"current": 3085.363,
"min": 0.0,
"max": 0.0
},
{
"current": 3176.23,
"min": 0.0,
"max": 0.0
},
{
"current": 3098.413,
"min": 0.0,
"max": 0.0
},
{
"current": 3131.838,
"min": 0.0,
"max": 0.0
},
{
"current": 3297.418,
"min": 0.0,
"max": 0.0
},
{
"current": 3144.573,
"min": 0.0,
"max": 0.0
},
{
"current": 3142.177,
"min": 0.0,
"max": 0.0
},
{
"current": 3135.089,
"min": 0.0,
"max": 0.0
},
{
"current": 3124.315,
"min": 0.0,
"max": 0.0
},
{
"current": 3206.745,
"min": 0.0,
"max": 0.0
},
{
"current": 3197.608,
"min": 0.0,
"max": 0.0
},
{
"current": 3271.659,
"min": 0.0,
"max": 0.0
},
{
"current": 3055.483,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.813,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.316,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.471,
"min": 0.0,
"max": 0.0
},
{
"current": 3275.344,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.318,
"min": 0.0,
"max": 0.0
},
{
"current": 3272.185,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.032,
"min": 0.0,
"max": 0.0
},
{
"current": 3273.055,
"min": 0.0,
"max": 0.0
},
{
"current": 3277.573,
"min": 0.0,
"max": 0.0
},
{
"current": 3274.44,
"min": 0.0,
"max": 0.0
},
{
"current": 3275.925,
"min": 0.0,
"max": 0.0
},
{
"current": 3279.092,
"min": 0.0,
"max": 0.0
},
{
"current": 3275.089,
"min": 0.0,
"max": 0.0
},
{
"current": 3277.671,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.135,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.31,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.038,
"min": 0.0,
"max": 0.0
},
{
"current": 3218.557,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.859,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.545,
"min": 0.0,
"max": 0.0
},
{
"current": 3027.843,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.687,
"min": 0.0,
"max": 0.0
},
{
"current": 3053.229,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.26,
"min": 0.0,
"max": 0.0
},
{
"current": 3059.862,
"min": 0.0,
"max": 0.0
},
{
"current": 3090.937,
"min": 0.0,
"max": 0.0
},
{
"current": 3094.897,
"min": 0.0,
"max": 0.0
},
{
"current": 3083.774,
"min": 0.0,
"max": 0.0
},
{
"current": 3027.722,
"min": 0.0,
"max": 0.0
},
{
"current": 3303.02,
"min": 0.0,
"max": 0.0
},
{
"current": 3069.951,
"min": 0.0,
"max": 0.0
},
{
"current": 3049.694,
"min": 0.0,
"max": 0.0
},
{
"current": 2814.624,
"min": 0.0,
"max": 0.0
},
{
"current": 3097.913,
"min": 0.0,
"max": 0.0
},
{
"current": 2788.423,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.195,
"min": 0.0,
"max": 0.0
},
{
"current": 3069.533,
"min": 0.0,
"max": 0.0
},
{
"current": 3074.679,
"min": 0.0,
"max": 0.0
},
{
"current": 3066.308,
"min": 0.0,
"max": 0.0
},
{
"current": 2598.471,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.109,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.455,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.945,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.926,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.003,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.535,
"min": 0.0,
"max": 0.0
}
],
"disk": {
"/": {
"total": 32.0,
"used": 0.01256561279296875
}
},
"gpu": "NVIDIA A10G",
"gpu_count": 8,
"gpu_devices": [
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
}
],
"memory": {
"total": 747.9597625732422
}
}