Training in progress, epoch 0

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,30 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652,
+  "[/REVISION]": 151670,
+  "[/STEP]": 151666,
+  "[/TURN]": 151668,
+  "[REVISION]": 151669,
+  "[STEP]": 151665,
+  "[TURN]": 151667
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151671
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a98b4e3194379ba7bec05ef5af0baa479391fb72c51b261749bdd1d251fb9b81
+size 4874843752

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab9e2130ff05c8efc6ce715675f96987ccecebb141684c969aa92ece6777278a
+size 4932751008

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:181225151db82d7c2a4b0d6bf1b800f2aa0b9e19ebcbc3f666e96ff341cd3a0b
+size 4330865200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9fd93165bb07b6dbb7f816180d4f65ad089d8c4efe19e5171a47c0c751f84b0
+size 1087177856

model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 15225598976
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,60 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "[STEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[/STEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[TURN]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[/TURN]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[REVISION]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[/REVISION]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:119474abe8593f47402f00c90e22eac0436e64f8c84c8270309a27e152901418
+size 11423104

tokenizer_config.json

@@ -0,0 +1,248 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "[STEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "[/STEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "[TURN]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "[/TURN]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "[REVISION]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "[/REVISION]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "[STEP]",
+    "[/STEP]",
+    "[TURN]",
+    "[/TURN]",
+    "[REVISION]",
+    "[/REVISION]"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d8b58b2fd3affe635bf267c95f88ed337f48f927e28276811b6aa45b9a3dd53
+size 6328

wandb/debug-internal.log

@@ -0,0 +1,373 @@
+2024-11-13 19:02:46,431 INFO    StreamThr :1939 [internal.py:wandb_internal():86] W&B internal server running at pid: 1939, started at: 2024-11-13 19:02:46.431351
+2024-11-13 19:02:46,433 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status
+2024-11-13 19:02:46,434 INFO    WriterThread:1939 [datastore.py:open_for_write():87] open: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/run-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1.wandb
+2024-11-13 19:02:46,435 DEBUG   SenderThread:1939 [sender.py:send():382] send: header
+2024-11-13 19:02:46,442 DEBUG   SenderThread:1939 [sender.py:send():382] send: run
+2024-11-13 19:02:46,772 INFO    SenderThread:1939 [dir_watcher.py:__init__():211] watching files in: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files
+2024-11-13 19:02:46,772 INFO    SenderThread:1939 [sender.py:_start_run_threads():1136] run started: kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1 with start time 1731524566.431221
+2024-11-13 19:02:46,782 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: check_version
+2024-11-13 19:02:46,782 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: check_version
+2024-11-13 19:02:46,859 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: run_start
+2024-11-13 19:02:46,947 DEBUG   HandlerThread:1939 [system_info.py:__init__():27] System info init
+2024-11-13 19:02:46,947 DEBUG   HandlerThread:1939 [system_info.py:__init__():42] System info init done
+2024-11-13 19:02:46,947 INFO    HandlerThread:1939 [system_monitor.py:start():194] Starting system monitor
+2024-11-13 19:02:46,947 INFO    SystemMonitor:1939 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-11-13 19:02:46,948 INFO    HandlerThread:1939 [system_monitor.py:probe():214] Collecting system info
+2024-11-13 19:02:46,948 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started cpu monitoring
+2024-11-13 19:02:46,949 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started disk monitoring
+2024-11-13 19:02:46,949 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started gpu monitoring
+2024-11-13 19:02:46,951 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started memory monitoring
+2024-11-13 19:02:46,952 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started network monitoring
+2024-11-13 19:02:47,010 DEBUG   HandlerThread:1939 [system_info.py:probe():151] Probing system
+2024-11-13 19:02:47,013 DEBUG   HandlerThread:1939 [system_info.py:_probe_git():136] Probing git
+2024-11-13 19:02:47,021 DEBUG   HandlerThread:1939 [system_info.py:_probe_git():144] Probing git done
+2024-11-13 19:02:47,021 DEBUG   HandlerThread:1939 [system_info.py:probe():199] Probing system done
+2024-11-13 19:02:47,022 DEBUG   HandlerThread:1939 [system_monitor.py:probe():223] {'os': 'Linux-5.10.226-214.879.amzn2.x86_64-x86_64-with-glibc2.31', 'python': '3.10.12', 'heartbeatAt': '2024-11-13T19:02:47.010671', 'startedAt': '2024-11-13T19:02:46.426230', 'docker': None, 'cuda': None, 'args': ('--batch_size', '64', '--cache_dir', '/opt/ml/data/input/.cache', '--dataset', 'code-contests', '--gradient_accumulation_steps', '8', '--logging_steps', '5', '--lr', '1e-6', '--max_seq_length', '2048', '--model_name', 'Qwen/Qwen2.5-Coder-7B-Instruct', '--num_train_epochs', '2', '--output_dir', '/opt/ml/model/', '--push_to_hub', 'True', '--run_name', 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', '--save_steps', '100', '--schedule', 'cosine', '--wandb_project', 'sft-codecontests-1112', '--weight_decay', '0.0'), 'state': 'running', 'program': '/opt/ml/code/qlearning_reasoning/training/sft.py', 'codePathLocal': 'qlearning_reasoning/training/sft.py', 'codePath': 'qlearning_reasoning/training/sft.py', 'git': {'remote': 'git@github.com:TRI-ML/reasoning-value-verifiers.git', 'commit': 'cbfd05147f15c55be83f4996eebdb305c17f119d'}, 'email': None, 'root': '/opt/ml/code', 'host': 'algo-1', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 96, 'cpu_count_logical': 192, 'cpu_freq': {'current': 2829.910838541667, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2850.788, 'min': 0.0, 'max': 0.0}, {'current': 3406.159, 'min': 0.0, 'max': 0.0}, {'current': 2605.944, 'min': 0.0, 'max': 0.0}, {'current': 2608.162, 'min': 0.0, 'max': 0.0}, {'current': 2818.169, 'min': 0.0, 'max': 0.0}, {'current': 2591.551, 'min': 0.0, 'max': 0.0}, {'current': 2621.486, 'min': 0.0, 'max': 0.0}, {'current': 2607.504, 'min': 0.0, 'max': 0.0}, {'current': 2999.734, 'min': 0.0, 'max': 0.0}, {'current': 2991.449, 'min': 0.0, 'max': 0.0}, {'current': 3123.177, 'min': 0.0, 'max': 0.0}, {'current': 3053.641, 'min': 0.0, 'max': 0.0}, {'current': 3003.014, 'min': 0.0, 'max': 0.0}, {'current': 3000.178, 'min': 0.0, 'max': 0.0}, {'current': 3432.256, 'min': 0.0, 'max': 0.0}, {'current': 2999.802, 'min': 0.0, 'max': 0.0}, {'current': 3005.725, 'min': 0.0, 'max': 0.0}, {'current': 3565.885, 'min': 0.0, 'max': 0.0}, {'current': 2992.339, 'min': 0.0, 'max': 0.0}, {'current': 3104.468, 'min': 0.0, 'max': 0.0}, {'current': 3182.411, 'min': 0.0, 'max': 0.0}, {'current': 3013.217, 'min': 0.0, 'max': 0.0}, {'current': 3131.904, 'min': 0.0, 'max': 0.0}, {'current': 3007.314, 'min': 0.0, 'max': 0.0}, {'current': 2625.175, 'min': 0.0, 'max': 0.0}, {'current': 2653.407, 'min': 0.0, 'max': 0.0}, {'current': 2610.871, 'min': 0.0, 'max': 0.0}, {'current': 2793.855, 'min': 0.0, 'max': 0.0}, {'current': 2619.232, 'min': 0.0, 'max': 0.0}, {'current': 2627.886, 'min': 0.0, 'max': 0.0}, {'current': 2613.748, 'min': 0.0, 'max': 0.0}, {'current': 2318.039, 'min': 0.0, 'max': 0.0}, {'current': 3546.661, 'min': 0.0, 'max': 0.0}, {'current': 3022.56, 'min': 0.0, 'max': 0.0}, {'current': 3035.28, 'min': 0.0, 'max': 0.0}, {'current': 3541.443, 'min': 0.0, 'max': 0.0}, {'current': 3163.55, 'min': 0.0, 'max': 0.0}, {'current': 3023.79, 'min': 0.0, 'max': 0.0}, {'current': 3033.204, 'min': 0.0, 'max': 0.0}, {'current': 3031.695, 'min': 0.0, 'max': 0.0}, {'current': 3034.587, 'min': 0.0, 'max': 0.0}, {'current': 3031.71, 'min': 0.0, 'max': 0.0}, {'current': 3029.116, 'min': 0.0, 'max': 0.0}, {'current': 3293.701, 'min': 0.0, 'max': 0.0}, {'current': 3033.78, 'min': 0.0, 'max': 0.0}, {'current': 3032.034, 'min': 0.0, 'max': 0.0}, {'current': 3057.365, 'min': 0.0, 'max': 0.0}, {'current': 3028.183, 'min': 0.0, 'max': 0.0}, {'current': 3596.439, 'min': 0.0, 'max': 0.0}, {'current': 3327.853, 'min': 0.0, 'max': 0.0}, {'current': 3226.463, 'min': 0.0, 'max': 0.0}, {'current': 3259.361, 'min': 0.0, 'max': 0.0}, {'current': 3284.076, 'min': 0.0, 'max': 0.0}, {'current': 3257.698, 'min': 0.0, 'max': 0.0}, {'current': 3280.289, 'min': 0.0, 'max': 0.0}, {'current': 3258.834, 'min': 0.0, 'max': 0.0}, {'current': 2517.346, 'min': 0.0, 'max': 0.0}, {'current': 2553.091, 'min': 0.0, 'max': 0.0}, {'current': 2552.693, 'min': 0.0, 'max': 0.0}, {'current': 2689.091, 'min': 0.0, 'max': 0.0}, {'current': 2503.301, 'min': 0.0, 'max': 0.0}, {'current': 2524.4, 'min': 0.0, 'max': 0.0}, {'current': 2521.768, 'min': 0.0, 'max': 0.0}, {'current': 2530.364, 'min': 0.0, 'max': 0.0}, {'current': 3258.225, 'min': 0.0, 'max': 0.0}, {'current': 3597.406, 'min': 0.0, 'max': 0.0}, {'current': 3279.658, 'min': 0.0, 'max': 0.0}, {'current': 3292.257, 'min': 0.0, 'max': 0.0}, {'current': 3265.351, 'min': 0.0, 'max': 0.0}, {'current': 3286.715, 'min': 0.0, 'max': 0.0}, {'current': 3268.777, 'min': 0.0, 'max': 0.0}, {'current': 3242.068, 'min': 0.0, 'max': 0.0}, {'current': 3288.908, 'min': 0.0, 'max': 0.0}, {'current': 3278.645, 'min': 0.0, 'max': 0.0}, {'current': 3215.578, 'min': 0.0, 'max': 0.0}, {'current': 3327.101, 'min': 0.0, 'max': 0.0}, {'current': 3357.075, 'min': 0.0, 'max': 0.0}, {'current': 3144.838, 'min': 0.0, 'max': 0.0}, {'current': 3005.517, 'min': 0.0, 'max': 0.0}, {'current': 3597.381, 'min': 0.0, 'max': 0.0}, {'current': 3596.175, 'min': 0.0, 'max': 0.0}, {'current': 3317.258, 'min': 0.0, 'max': 0.0}, {'current': 3310.249, 'min': 0.0, 'max': 0.0}, {'current': 3140.687, 'min': 0.0, 'max': 0.0}, {'current': 3398.954, 'min': 0.0, 'max': 0.0}, {'current': 3318.433, 'min': 0.0, 'max': 0.0}, {'current': 3342.35, 'min': 0.0, 'max': 0.0}, {'current': 3328.575, 'min': 0.0, 'max': 0.0}, {'current': 3499.892, 'min': 0.0, 'max': 0.0}, {'current': 3344.864, 'min': 0.0, 'max': 0.0}, {'current': 3355.802, 'min': 0.0, 'max': 0.0}, {'current': 3305.032, 'min': 0.0, 'max': 0.0}, {'current': 3318.07, 'min': 0.0, 'max': 0.0}, {'current': 3336.546, 'min': 0.0, 'max': 0.0}, {'current': 3335.458, 'min': 0.0, 'max': 0.0}, {'current': 3596.561, 'min': 0.0, 'max': 0.0}, {'current': 2884.473, 'min': 0.0, 'max': 0.0}, {'current': 2934.098, 'min': 0.0, 'max': 0.0}, {'current': 2621.598, 'min': 0.0, 'max': 0.0}, {'current': 2645.184, 'min': 0.0, 'max': 0.0}, {'current': 2480.726, 'min': 0.0, 'max': 0.0}, {'current': 2515.912, 'min': 0.0, 'max': 0.0}, {'current': 2702.57, 'min': 0.0, 'max': 0.0}, {'current': 2671.473, 'min': 0.0, 'max': 0.0}, {'current': 3092.321, 'min': 0.0, 'max': 0.0}, {'current': 3092.204, 'min': 0.0, 'max': 0.0}, {'current': 3217.287, 'min': 0.0, 'max': 0.0}, {'current': 3161.646, 'min': 0.0, 'max': 0.0}, {'current': 3089.273, 'min': 0.0, 'max': 0.0}, {'current': 3088.447, 'min': 0.0, 'max': 0.0}, {'current': 3559.214, 'min': 0.0, 'max': 0.0}, {'current': 3218.611, 'min': 0.0, 'max': 0.0}, {'current': 3021.622, 'min': 0.0, 'max': 0.0}, {'current': 3332.084, 'min': 0.0, 'max': 0.0}, {'current': 3019.854, 'min': 0.0, 'max': 0.0}, {'current': 3213.831, 'min': 0.0, 'max': 0.0}, {'current': 3092.774, 'min': 0.0, 'max': 0.0}, {'current': 3015.868, 'min': 0.0, 'max': 0.0}, {'current': 3082.752, 'min': 0.0, 'max': 0.0}, {'current': 3029.559, 'min': 0.0, 'max': 0.0}, {'current': 2873.142, 'min': 0.0, 'max': 0.0}, {'current': 2215.288, 'min': 0.0, 'max': 0.0}, {'current': 3193.443, 'min': 0.0, 'max': 0.0}, {'current': 2928.717, 'min': 0.0, 'max': 0.0}, {'current': 2990.366, 'min': 0.0, 'max': 0.0}, {'current': 2923.847, 'min': 0.0, 'max': 0.0}, {'current': 2869.364, 'min': 0.0, 'max': 0.0}, {'current': 2859.189, 'min': 0.0, 'max': 0.0}, {'current': 3049.001, 'min': 0.0, 'max': 0.0}, {'current': 2805.206, 'min': 0.0, 'max': 0.0}, {'current': 3094.081, 'min': 0.0, 'max': 0.0}, {'current': 3187.222, 'min': 0.0, 'max': 0.0}, {'current': 2853.945, 'min': 0.0, 'max': 0.0}, {'current': 2944.73, 'min': 0.0, 'max': 0.0}, {'current': 2809.161, 'min': 0.0, 'max': 0.0}, {'current': 2806.787, 'min': 0.0, 'max': 0.0}, {'current': 3024.225, 'min': 0.0, 'max': 0.0}, {'current': 3032.173, 'min': 0.0, 'max': 0.0}, {'current': 3033.195, 'min': 0.0, 'max': 0.0}, {'current': 3561.287, 'min': 0.0, 'max': 0.0}, {'current': 2987.872, 'min': 0.0, 'max': 0.0}, {'current': 3051.252, 'min': 0.0, 'max': 0.0}, {'current': 3379.017, 'min': 0.0, 'max': 0.0}, {'current': 3044.319, 'min': 0.0, 'max': 0.0}, {'current': 3595.62, 'min': 0.0, 'max': 0.0}, {'current': 3261.737, 'min': 0.0, 'max': 0.0}, {'current': 3257.013, 'min': 0.0, 'max': 0.0}, {'current': 3288.317, 'min': 0.0, 'max': 0.0}, {'current': 3268.675, 'min': 0.0, 'max': 0.0}, {'current': 3269.579, 'min': 0.0, 'max': 0.0}, {'current': 3266.213, 'min': 0.0, 'max': 0.0}, {'current': 3265.148, 'min': 0.0, 'max': 0.0}, {'current': 2554.152, 'min': 0.0, 'max': 0.0}, {'current': 2556.63, 'min': 0.0, 'max': 0.0}, {'current': 2540.571, 'min': 0.0, 'max': 0.0}, {'current': 2512.005, 'min': 0.0, 'max': 0.0}, {'current': 2560.992, 'min': 0.0, 'max': 0.0}, {'current': 2526.641, 'min': 0.0, 'max': 0.0}, {'current': 2528.267, 'min': 0.0, 'max': 0.0}, {'current': 2583.061, 'min': 0.0, 'max': 0.0}, {'current': 3257.652, 'min': 0.0, 'max': 0.0}, {'current': 3598.217, 'min': 0.0, 'max': 0.0}, {'current': 3282.578, 'min': 0.0, 'max': 0.0}, {'current': 3261.664, 'min': 0.0, 'max': 0.0}, {'current': 3268.246, 'min': 0.0, 'max': 0.0}, {'current': 3331.937, 'min': 0.0, 'max': 0.0}, {'current': 3265.603, 'min': 0.0, 'max': 0.0}, {'current': 3258.501, 'min': 0.0, 'max': 0.0}, {'current': 3337.045, 'min': 0.0, 'max': 0.0}, {'current': 3259.949, 'min': 0.0, 'max': 0.0}, {'current': 3226.058, 'min': 0.0, 'max': 0.0}, {'current': 3250.308, 'min': 0.0, 'max': 0.0}, {'current': 3208.026, 'min': 0.0, 'max': 0.0}, {'current': 2974.043, 'min': 0.0, 'max': 0.0}, {'current': 3216.706, 'min': 0.0, 'max': 0.0}, {'current': 3598.55, 'min': 0.0, 'max': 0.0}, {'current': 3598.261, 'min': 0.0, 'max': 0.0}, {'current': 3222.637, 'min': 0.0, 'max': 0.0}, {'current': 3227.662, 'min': 0.0, 'max': 0.0}, {'current': 3143.781, 'min': 0.0, 'max': 0.0}, {'current': 3215.567, 'min': 0.0, 'max': 0.0}, {'current': 3250.612, 'min': 0.0, 'max': 0.0}, {'current': 3195.784, 'min': 0.0, 'max': 0.0}, {'current': 3219.289, 'min': 0.0, 'max': 0.0}, {'current': 3289.163, 'min': 0.0, 'max': 0.0}, {'current': 3247.875, 'min': 0.0, 'max': 0.0}, {'current': 3249.112, 'min': 0.0, 'max': 0.0}, {'current': 3209.3, 'min': 0.0, 'max': 0.0}, {'current': 3187.212, 'min': 0.0, 'max': 0.0}, {'current': 3288.003, 'min': 0.0, 'max': 0.0}, {'current': 3169.417, 'min': 0.0, 'max': 0.0}, {'current': 3596.071, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 119.94140625, 'used': 56.4015998840332}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9661331176758}}
+2024-11-13 19:02:47,022 INFO    HandlerThread:1939 [system_monitor.py:probe():224] Finished collecting system info
+2024-11-13 19:02:47,022 INFO    HandlerThread:1939 [system_monitor.py:probe():227] Publishing system info
+2024-11-13 19:02:47,022 DEBUG   HandlerThread:1939 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment
+2024-11-13 19:02:47,775 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/conda-environment.yaml
+2024-11-13 19:03:01,345 DEBUG   HandlerThread:1939 [system_info.py:_save_conda():220] Saving conda packages done
+2024-11-13 19:03:01,345 DEBUG   HandlerThread:1939 [system_info.py:_save_code():45] Saving code
+2024-11-13 19:03:01,349 DEBUG   HandlerThread:1939 [system_info.py:_save_code():66] Saving code done
+2024-11-13 19:03:01,349 DEBUG   HandlerThread:1939 [system_info.py:_save_patches():83] Saving git patches
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/conda-environment.yaml
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/code/qlearning_reasoning/training/sft.py
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/code/qlearning_reasoning
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/code/qlearning_reasoning/training
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/code
+2024-11-13 19:03:01,989 DEBUG   HandlerThread:1939 [system_info.py:_save_patches():125] Saving git patches done
+2024-11-13 19:03:01,991 INFO    HandlerThread:1939 [system_monitor.py:probe():229] Finished publishing system info
+2024-11-13 19:03:01,993 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:01,993 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: keepalive
+2024-11-13 19:03:01,994 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:01,994 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: keepalive
+2024-11-13 19:03:01,994 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:01,994 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: keepalive
+2024-11-13 19:03:01,994 DEBUG   SenderThread:1939 [sender.py:send():382] send: files
+2024-11-13 19:03:01,995 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-11-13 19:03:01,995 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file code/qlearning_reasoning/training/sft.py with policy now
+2024-11-13 19:03:01,995 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file diff.patch with policy now
+2024-11-13 19:03:01,999 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: python_packages
+2024-11-13 19:03:01,999 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: python_packages
+2024-11-13 19:03:02,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:03:02,003 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:03:02,003 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:03:02,111 DEBUG   SenderThread:1939 [sender.py:send():382] send: telemetry
+2024-11-13 19:03:02,111 DEBUG   SenderThread:1939 [sender.py:send():382] send: config
+2024-11-13 19:03:02,112 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:02,112 DEBUG   SenderThread:1939 [sender.py:send():382] send: telemetry
+2024-11-13 19:03:02,112 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:02,113 WARNING SenderThread:1939 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen)
+2024-11-13 19:03:02,113 DEBUG   SenderThread:1939 [sender.py:send():382] send: telemetry
+2024-11-13 19:03:02,113 DEBUG   SenderThread:1939 [sender.py:send():382] send: telemetry
+2024-11-13 19:03:02,113 DEBUG   SenderThread:1939 [sender.py:send():382] send: config
+2024-11-13 19:03:02,551 INFO    wandb-upload_1:1939 [upload_job.py:push():131] Uploaded file /tmp/tmpm_h7h9v0wandb/pjhigh2e-code/qlearning_reasoning/training/sft.py
+2024-11-13 19:03:02,554 INFO    wandb-upload_0:1939 [upload_job.py:push():131] Uploaded file /tmp/tmpm_h7h9v0wandb/f98paeu8-wandb-metadata.json
+2024-11-13 19:03:02,556 INFO    wandb-upload_2:1939 [upload_job.py:push():131] Uploaded file /tmp/tmpm_h7h9v0wandb/cc8zuviy-diff.patch
+2024-11-13 19:03:02,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/diff.patch
+2024-11-13 19:03:02,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:03:02,777 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/requirements.txt
+2024-11-13 19:03:02,777 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-metadata.json
+2024-11-13 19:03:04,777 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:03:07,114 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:12,115 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:16,999 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:03:17,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:03:17,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:03:18,090 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:18,782 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/config.yaml
+2024-11-13 19:03:23,250 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:26,095 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:03:26,098 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:03:26,099 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:03:26,785 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:03:28,785 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:03:29,100 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:31,999 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:03:32,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:03:32,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:03:34,147 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:39,147 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:44,148 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:46,955 DEBUG   SystemMonitor:1939 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-11-13 19:03:46,959 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:03:47,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:03:47,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:03:47,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:03:49,636 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:50,813 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:03:50,814 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/config.yaml
+2024-11-13 19:03:54,814 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:59,814 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:02,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:04:02,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:04:02,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:04:05,061 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:10,062 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:14,822 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:04:15,419 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:16,962 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:04:17,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:04:17,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:04:17,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:04:21,087 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:26,088 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:31,089 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:32,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:04:32,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:04:32,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:04:36,587 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:36,829 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:04:41,588 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:46,588 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:46,965 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:04:47,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:04:47,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:04:47,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:04:52,111 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:57,111 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:57,383 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:04:57,384 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:04:57,385 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:04:57,385 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:04:57,837 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:04:58,837 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:04:59,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:04:59,004 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:04:59,005 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:04:59,005 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:04:59,005 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:04:59,006 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:04:59,006 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:04:59,006 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:04:59,838 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:05:00,838 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:05:02,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:05:02,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:05:02,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:05:03,111 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:08,112 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:13,113 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:16,967 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:05:17,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:05:17,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:05:17,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:05:18,123 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:22,846 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:05:23,982 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:25,847 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/config.yaml
+2024-11-13 19:05:30,486 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:32,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:05:32,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:05:32,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:05:36,117 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:41,118 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:46,513 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:46,854 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:05:46,970 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:05:47,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:05:47,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:05:47,003 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:05:52,097 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:57,098 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:02,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:06:02,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:06:02,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:06:02,152 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:07,153 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:08,862 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:06:12,639 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:16,973 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:06:17,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:06:17,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:06:17,004 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:06:18,127 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:23,127 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:28,128 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:30,870 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:06:32,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:06:32,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:06:32,003 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:06:33,143 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:38,144 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:43,144 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:46,975 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:06:47,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:06:47,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:06:47,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:06:49,137 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:51,931 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:06:51,932 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:06:51,933 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:06:51,934 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:06:52,878 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:06:52,878 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:06:53,541 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:06:53,543 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:06:53,543 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:06:53,544 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:06:53,878 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:06:54,545 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:54,878 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:06:59,546 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:02,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:07:02,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:07:02,004 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:07:05,164 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:10,165 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:15,165 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:16,886 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:07:16,978 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:07:17,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:07:17,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:07:17,003 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:07:21,108 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:26,108 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:31,109 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:32,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:07:32,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:07:32,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:07:36,934 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:38,894 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:07:41,935 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:46,935 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:46,981 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:07:47,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:07:47,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:07:47,004 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:07:52,135 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:57,136 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:02,136 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:02,856 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:08:02,923 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:08:02,924 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:08:08,076 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:13,077 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:16,985 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:08:17,407 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:08:17,407 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:08:17,447 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:08:18,545 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:23,546 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:28,546 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:32,914 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:33,843 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:34,729 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:08:34,729 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:08:34,771 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:08:39,802 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:44,803 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:46,987 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:08:48,007 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:08:48,007 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:08:48,008 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:08:48,921 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:49,945 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:50,922 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:52,923 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:54,924 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:55,154 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:56,925 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:58,926 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:09:00,371 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:00,948 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:09:03,950 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:09:05,541 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:05,782 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:09:06,102 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:09:06,102 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:09:11,263 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:16,264 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:16,990 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:09:20,961 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:09:22,073 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:22,272 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:09:22,272 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:09:27,401 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:32,401 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:36,126 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:09:36,666 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:09:36,666 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:09:38,052 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:39,964 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:09:43,769 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:46,992 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:09:47,846 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:09:47,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:09:47,847 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:09:49,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:54,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:59,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:02,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:10:02,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:10:02,848 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:10:05,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:10,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:11,977 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:15,248 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:16,996 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:10:17,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:10:17,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:10:17,848 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:10:21,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:26,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:31,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:32,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:10:32,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:10:32,847 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:10:33,985 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:37,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:42,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:46,998 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:10:47,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:10:47,847 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:10:47,887 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:10:47,921 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:53,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:53,575 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:10:53,576 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:10:53,576 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:10:53,577 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:10:53,992 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:10:54,993 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:55,200 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:10:55,201 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:10:55,201 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:10:55,203 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:10:55,993 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:55,994 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:10:56,994 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:57,994 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:59,204 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:02,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:11:02,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:11:02,849 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:11:04,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:09,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:14,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:17,000 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:11:17,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:11:17,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:11:17,848 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:11:19,311 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:20,004 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:11:24,312 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:29,312 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:32,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:11:32,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:11:32,849 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:11:35,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:40,232 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:44,014 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:11:46,173 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:47,003 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:11:47,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:11:47,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:11:47,848 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:11:51,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report

wandb/debug.log

@@ -0,0 +1,30 @@
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Configure stats pid to 173
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Loading settings from /opt/ml/code/wandb/settings
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'api_key': '***REDACTED***', 'project': 'sft-codecontests-1112', 'run_name': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048'}
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'qlearning_reasoning/training/sft.py', 'program_abspath': '/opt/ml/code/qlearning_reasoning/training/sft.py', 'program': '/opt/ml/code/qlearning_reasoning/training/sft.py'}
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:_log_setup():526] Logging user logs to /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/logs/debug.log
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:_log_setup():527] Logging internal logs to /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/logs/debug-internal.log
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:init():566] calling init triggers
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sagemaker_training_job_name': 'kushalarora-rvv-main-2024-11-13-17-55-42-664', 'batch_size': 64, 'cache_dir': '/opt/ml/data/input/.cache', 'dataset': 'code-contests', 'gradient_accumulation_steps': 8, 'logging_steps': 5, 'lr': '1e-6', 'max_seq_length': 2048, 'model_name': 'Qwen/Qwen2.5-Coder-7B-Instruct', 'num_train_epochs': 2, 'output_dir': '/opt/ml/output', 'push_to_hub': 'true', 'run_name': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'sagemaker_container_log_level': 20, 'sagemaker_instance_type': 'ml.p5.48xlarge', 'sagemaker_job_name': 'kushalarora-rvv-main-2024-11-13-17-55-42-664', 'sagemaker_program': 'sft.py', 'sagemaker_region': 'us-east-1', 'sagemaker_submit_directory': 's3://tri-ml-datasets/kushalarora-rvv-main-2024-11-13-17-55-42-664/source/sourcedir.tar.gz', 'sagemaker_torch_distributed_enabled': 'true', 'save_steps': 100, 'schedule': 'cosine', 'wandb_project': 'sft-codecontests-1112', 'weight_decay': 0.0, 'additional_framework_parameters': {'sagemaker_instance_type': 'ml.p5.48xlarge', 'sagemaker_torch_distributed_enabled': True}, 'channel_input_dirs': {}, 'current_host': 'algo-1', 'current_instance_group': 'homogeneousCluster', 'current_instance_group_hosts': ['algo-1'], 'current_instance_type': 'ml.p5.48xlarge', 'distribution_hosts': ['algo-1'], 'distribution_instance_groups': ['homogeneousCluster'], 'framework_module': 'sagemaker_pytorch_container.training:main', 'hosts': ['algo-1'], 'hyperparameters': {'batch_size': 64, 'cache_dir': '/opt/ml/data/input/.cache', 'dataset': 'code-contests', 'gradient_accumulation_steps': 8, 'logging_steps': 5, 'lr': '1e-6', 'max_seq_length': 2048, 'model_name': 'Qwen/Qwen2.5-Coder-7B-Instruct', 'num_train_epochs': 2, 'output_dir': '/opt/ml/model/', 'push_to_hub': True, 'run_name': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'save_steps': 100, 'schedule': 'cosine', 'wandb_project': 'sft-codecontests-1112', 'weight_decay': 0.0}, 'input_config_dir': '/opt/ml/input/config', 'input_data_config': {}, 'input_dir': '/opt/ml/input', 'instance_groups': ['homogeneousCluster'], 'instance_groups_dict': {'homogeneousCluster': {'hosts': ['algo-1'], 'instance_group_name': 'homogeneousCluster', 'instance_type': 'ml.p5.48xlarge'}}, 'is_hetero': False, 'is_master': True, 'is_modelparallel_enabled': None, 'is_smddpmprun_installed': False, 'is_smddprun_installed': True, 'job_name': 'kushalarora-rvv-main-2024-11-13-17-55-42-664', 'log_level': 20, 'master_hostname': 'algo-1', 'model_dir': '/opt/ml/model', 'module_dir': 's3://tri-ml-datasets/kushalarora-rvv-main-2024-11-13-17-55-42-664/source/sourcedir.tar.gz', 'module_name': '/opt/ml/code/qlearning_reasoning/training/sft', 'network_interface_name': 'eth0', 'num_cpus': 192, 'num_gpus': 8, 'num_neurons': 0, 'output_data_dir': '/opt/ml/output/data', 'output_intermediate_dir': '/opt/ml/output/intermediate', 'resource_config': {'current_group_name': 'homogeneousCluster', 'current_host': 'algo-1', 'current_instance_type': 'ml.p5.48xlarge', 'hosts': ['algo-1'], 'instance_groups': [{'hosts': ['algo-1'], 'instance_group_name': 'homogeneousCluster', 'instance_type': 'ml.p5.48xlarge'}], 'network_interface_name': 'eth0'}, 'user_entry_point': '/opt/ml/code/qlearning_reasoning/training/sft.py'}
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:init():616] starting backend
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:init():620] setting up manager
+2024-11-13 19:02:46,429 INFO    MainThread:173 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-13 19:02:46,431 INFO    MainThread:173 [wandb_init.py:init():628] backend started and connected
+2024-11-13 19:02:46,434 INFO    MainThread:173 [wandb_init.py:init():720] updated telemetry
+2024-11-13 19:02:46,440 INFO    MainThread:173 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-11-13 19:02:46,781 INFO    MainThread:173 [wandb_run.py:_on_init():2262] communicating current version
+2024-11-13 19:02:46,812 INFO    MainThread:173 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.18.6 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-11-13 19:02:46,813 INFO    MainThread:173 [wandb_init.py:init():804] starting run threads in backend
+2024-11-13 19:03:02,000 INFO    MainThread:173 [wandb_run.py:_console_start():2241] atexit reg
+2024-11-13 19:03:02,000 INFO    MainThread:173 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-11-13 19:03:02,000 INFO    MainThread:173 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-11-13 19:03:02,001 INFO    MainThread:173 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-11-13 19:03:02,001 INFO    MainThread:173 [wandb_init.py:init():847] run started, returning control to user process
+2024-11-13 19:03:02,004 INFO    MainThread:173 [wandb_run.py:_config_callback():1343] config_cb None None {'vocab_size': 151671, 'max_position_embeddings': 32768, 'hidden_size': 3584, 'intermediate_size': 18944, 'num_hidden_layers': 28, 'num_attention_heads': 28, 'use_sliding_window': False, 'sliding_window': None, 'max_window_layers': 28, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 1000000.0, 'rope_scaling': None, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['Qwen2ForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': 151643, 'eos_token_id': 151645, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'Qwen/Qwen2.5-Coder-7B-Instruct', '_attn_implementation_autoset': True, 'transformers_version': '4.46.0', 'model_type': 'qwen2', 'output_dir': '/opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 8, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-06, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/runs/Nov13_19-02-15_algo-1', 'logging_strategy': 'steps', 'logging_first_step': True, 'logging_steps': 5, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 100, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': ['full_shard', 'auto_wrap'], 'fsdp_min_num_params': 0, 'fsdp_config': {'activation_checkpointing': False, 'auto_wrap_policy': 'TRANSFORMER_BASED_WRAP', 'backward_prefetch': 'BACKWARD_PRE', 'cpu_ram_efficient_loading': True, 'forward_prefetch': False, 'offload_params': False, 'sharding_strategy': 'FULL_SHARD', 'state_dict_type': 'FULL_STATE_DICT', 'sync_module_states': True, 'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': {'use_reentrant': False}, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False, 'dataset_text_field': 'text', 'packing': False, 'max_seq_length': 2048, 'dataset_num_proc': None, 'dataset_batch_size': 64, 'model_init_kwargs': None, 'dataset_kwargs': {}, 'eval_packing': None, 'num_of_sequences': 1024, 'chars_per_token': '<CHARS_PER_TOKEN>', 'use_liger': False}
+2024-11-13 19:03:02,008 INFO    MainThread:173 [wandb_config.py:__setitem__():151] config set model/num_parameters = 951599936 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7fb464957310>>
+2024-11-13 19:03:02,009 INFO    MainThread:173 [wandb_run.py:_config_callback():1343] config_cb model/num_parameters 951599936 None

wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/code/qlearning_reasoning/training/sft.py

@@ -0,0 +1,249 @@
+from datasets import load_dataset, concatenate_datasets
+from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments
+import torch
+from transformers import get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, get_linear_schedule_with_warmup, get_inverse_sqrt_schedule
+from trl import SFTTrainer, SFTConfig, DataCollatorForCompletionOnlyLM
+import os
+from textwrap import dedent
+from absl import app, flags
+FLAGS = flags.FLAGS
+import warnings
+warnings.filterwarnings("ignore")
+from trl import setup_chat_format
+
+ALL_DATASET_NAMES = [
+    'prm800k', 
+    'prm800k-improvement', 
+    'math', 
+    'math-v2', 
+    'math-prm800k', 
+    'math-revision', 
+    'og-math', 
+    'leetcode', 
+    'taco', 
+    'math-revision-onesamp', 
+    'math-revision-onesamp-overlap', 
+    'math-revision-onesamp-correct-overlap', 
+    'math-revision-onesamp-correct-overlap', 
+    'code-contests'
+]
+
+flags.DEFINE_enum('dataset', 'prm800k', ALL_DATASET_NAMES, 'Name of the dataset')
+flags.DEFINE_string('cache_dir', '/scratch/bclj/asingh15/.cache', 'Directory to store cache files')
+flags.DEFINE_string('output_dir', '/scratch/bclj/asingh15/sft_checkpoints/', 'Directory to store output files')
+flags.DEFINE_string('run_name', 'sft', 'Name of the run')
+flags.DEFINE_string('model_name', 'deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct', 'Name of the model')
+flags.DEFINE_string('token', None, 'Hugging Face API token')
+flags.DEFINE_string('wandb_project', 'sft-prm800k-pregen-0501', 'Wandb project name')
+flags.DEFINE_string('optim', 'adamw_torch', 'Optimizer')
+flags.DEFINE_string('schedule', 'constant', 'Learning rate schedule')
+flags.DEFINE_float('lr', 5e-5, 'Learning rate')
+flags.DEFINE_float('weight_decay', 0.0, 'Weight decay')
+flags.DEFINE_integer('warmup_steps', 0, 'Number of warmup steps')
+flags.DEFINE_integer('num_train_epochs', 1, 'Number of training epochs')
+flags.DEFINE_integer('batch_size', 256, 'Batch size')
+flags.DEFINE_integer('gradient_accumulation_steps', 64, 'Gradient accumulation steps')
+flags.DEFINE_integer('max_seq_length', 1024, 'Maximum sequence length')
+flags.DEFINE_integer('logging_steps', 100, 'Logging steps')
+flags.DEFINE_integer('save_steps', 1000, 'Save steps')
+flags.DEFINE_integer('seed', 42, 'Random seed')
+flags.DEFINE_string('fsdp', "full_shard auto_wrap", 'If to use FSDP')
+flags.DEFINE_string('fsdp_config', "qlearning_reasoning/accelerate_configs/fsdp_config.json", 'FSDP config to use.')
+flags.DEFINE_boolean("push_to_hub", False, "Should the model be uploaded to Huggingface hub?")
+flags.DEFINE_boolean('add_step_tokens', True, 'Add step tokens')
+flags.DEFINE_string("hub_model_id", None, "Hub model id. If not defined it will be truncated run name.")
+
+START_OF_STEP_TOKEN = '[STEP]'
+END_OF_STEP_TOKEN = '[/STEP]'
+START_OF_TURN_TOKEN = '[TURN]'
+END_OF_TURN_TOKEN = '[/TURN]'
+START_OF_REVISION_TOKEN = '[REVISION]'
+END_OF_REVISION_TOKEN = '[/REVISION]'
+
+def main(_):
+
+    if FLAGS.hub_model_id is None:
+        FLAGS.hub_model_id = FLAGS.run_name[:96]
+
+    if not os.path.exists(FLAGS.output_dir):
+        os.makedirs(FLAGS.output_dir, exist_ok=True)
+        os.makedirs(f"{FLAGS.output_dir}/{FLAGS.run_name}", exist_ok=True)
+
+    os.environ["WANDB_PROJECT"] = FLAGS.wandb_project
+    os.environ["WANDB_DIR"] = f"{FLAGS.output_dir}/{FLAGS.run_name}"
+    os.environ["WANDB_NAME"] = FLAGS.run_name
+    os.environ["HF_DATASETS_CACHE"] = FLAGS.cache_dir
+
+    shared_kwargs = dict(
+        cache_dir=FLAGS.cache_dir,
+        trust_remote_code=True,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        FLAGS.model_name,
+        **shared_kwargs,
+    )
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+    
+    if FLAGS.add_step_tokens:
+        tokenizer.add_special_tokens(
+            {'additional_special_tokens': [START_OF_STEP_TOKEN, END_OF_STEP_TOKEN, START_OF_TURN_TOKEN, END_OF_TURN_TOKEN, START_OF_REVISION_TOKEN, END_OF_REVISION_TOKEN]}
+        )
+
+        print(f"Tokenizer vocab size: {len(tokenizer)}")
+        print('EOS token:', tokenizer.eos_token)
+        print('PAD token:', tokenizer.pad_token)
+        print('Special tokens:', tokenizer.special_tokens_map)
+
+    instruction_template = "# Question\n\n"
+    if FLAGS.dataset == 'prm800k':
+        dataset_name = 'Asap7772/sft_prm800k_processed'
+        response_template = "# Solution\n\n"
+    elif FLAGS.dataset == 'math':
+        dataset_name = 'Asap7772/sft_math_processed'
+        response_template = "# Solution\n\n"
+    elif FLAGS.dataset == 'math-prm800k':
+        dataset_name = ['Asap7772/sft_prm800k_processed', 'Asap7772/sft_math_processed']
+        response_template = "# Solution\n\n"
+    elif FLAGS.dataset == 'og-math':
+        dataset_name = 'Asap7772/hendrycks_math_sft_formatted'
+        response_template = "# Solution\n\n"
+    elif FLAGS.dataset == 'leetcode':
+        dataset_name = 'Asap7772/leetcode-rosetta-processed'
+        instruction_template = "# Question"
+        response_template = "# Solution"
+    elif FLAGS.dataset == 'prm800k-improvement':
+        dataset_name = 'Asap7772/prm800k_improvement_groundtruth_values'
+        instruction_template = 'Solution Steps:'
+        response_template = "Improved Step:"
+    elif FLAGS.dataset == 'taco':
+        dataset_name = 'Asap7772/TACO-Processed'
+        instruction_template = "# Question"
+        response_template = "# Solution"
+    elif FLAGS.dataset == 'math-revision':
+        dataset_name = 'Asap7772/hendrycks-math-mc-llama-sftnoic-revisions-filtered'
+        instruction_template = "# Question"
+        response_template = f"{END_OF_TURN_TOKEN}\n\n{START_OF_TURN_TOKEN}"
+    elif FLAGS.dataset == 'math-revision-ic':
+        dataset_name = 'Asap7772/hendrycks-math-mc-llama-revisions-filtered'
+        instruction_template = "# Question"
+        response_template = f"{END_OF_TURN_TOKEN}\n\n{START_OF_TURN_TOKEN}"
+    elif FLAGS.dataset == 'math-revision-onesamp':
+        dataset_name = 'Asap7772/hendrycks-math-onesamp-revision'
+        instruction_template = "# Question"
+        response_template = "# Solution"
+    elif FLAGS.dataset == 'math-revision-onesamp-overlap':
+        dataset_name = 'Asap7772/hendrycks-math-onesamp-revision-overlap'
+        instruction_template = "# Question"
+        response_template = "# Solution"
+    elif FLAGS.dataset == 'math-v2':
+        dataset_name = 'Asap7772/sft_math_405b'
+        instruction_template = "# Question"
+        response_template = "# Solution"
+    elif FLAGS.dataset == 'math-revision-onesamp-correct-overlap':
+        dataset_name = 'Asap7772/hendrycks-math-onesamp-revision-correct-overlap'
+        instruction_template = "# Question"
+        response_template = "# Solution"
+    elif FLAGS.dataset == 'code-contests':
+        dataset_name = 'Asap7772/code_contests'
+        instruction_template = "Q:"
+        response_template = "A:"
+    else:
+        raise ValueError(f"Invalid dataset: {FLAGS.dataset}")
+
+    if isinstance(dataset_name, list):
+        all_datasets = [load_dataset(name, cache_dir=FLAGS.cache_dir) for name in dataset_name]
+        train_dataset = concatenate_datasets([dataset['train'] for dataset in all_datasets])
+        eval_dataset = concatenate_datasets([dataset['test'] for dataset in all_datasets])
+        print(f"example: {train_dataset[0]}")
+    elif isinstance(dataset_name, str):
+        dataset_dict = load_dataset(dataset_name, cache_dir=FLAGS.cache_dir)
+        train_dataset, eval_dataset = dataset_dict['train'], dataset_dict['test']
+    else:
+        raise ValueError(f"Invalid dataset name: {dataset_name}")
+    train_dataset, eval_dataset = train_dataset.shuffle(seed=FLAGS.seed), eval_dataset.shuffle(seed=FLAGS.seed)
+
+    num_train_epochs = FLAGS.num_train_epochs
+
+    def formatting_prompts_func(example):
+        output_texts = []
+        for i in range(len(example['text'])):
+            if FLAGS.dataset in ALL_DATASET_NAMES:
+                text = f"{example['text'][i]}{tokenizer.eos_token}"
+            else:
+                raise ValueError(f"Invalid dataset: {FLAGS.dataset}")
+
+            text = dedent(text).strip()
+            output_texts.append(text)
+        return output_texts
+    format_example = formatting_prompts_func(train_dataset[0:1])
+    print(f"Example formatted prompt: \n\n{format_example[0]}\n\n")
+    assert instruction_template in format_example[0], f"Instruction template not found in text."
+    assert response_template in format_example[0], f"Response template not found in text."
+    assert tokenizer.eos_token in format_example[0], f"EOS token not found in text."
+
+    instruction_template_ids = tokenizer.encode(instruction_template, add_special_tokens=False)[:2]
+    response_template_ids = tokenizer.encode(response_template, add_special_tokens=False)[:2]
+    collator = DataCollatorForCompletionOnlyLM(instruction_template=instruction_template_ids, response_template=response_template_ids, tokenizer=tokenizer, mlm=False)
+
+    model = AutoModelForCausalLM.from_pretrained(
+        FLAGS.model_name,
+        low_cpu_mem_usage=True,
+        torch_dtype=torch.bfloat16,
+        **shared_kwargs,
+    )
+    model.resize_token_embeddings(len(tokenizer))
+    if getattr(model, "config", None) is not None:
+        model.config.pad_token_id = tokenizer.pad_token_id
+        model.config.bos_token_id = tokenizer.bos_token_id
+        model.config.eos_token_id = tokenizer.eos_token_id
+    # Update the generation config to use the new eos & bos token
+    if getattr(model, "generation_config", None) is not None:
+        model.generation_config.bos_token_id = tokenizer.bos_token_id
+        model.generation_config.eos_token_id = tokenizer.eos_token_id
+        model.generation_config.pad_token_id = tokenizer.pad_token_id
+    
+    model.config.use_cache = False
+    model.gradient_checkpointing_enable()
+
+    transformer_args = SFTConfig(
+        run_name=FLAGS.run_name,
+        output_dir=f"{FLAGS.output_dir}/{FLAGS.run_name}",
+        learning_rate=FLAGS.lr,
+        eval_strategy="steps",
+        logging_strategy="steps",
+        logging_first_step=True,
+        save_strategy="epoch",
+        dataset_batch_size=FLAGS.batch_size,
+        max_seq_length=FLAGS.max_seq_length,
+        gradient_accumulation_steps=FLAGS.gradient_accumulation_steps,
+        gradient_checkpointing_kwargs={'use_reentrant':False},
+        logging_steps=FLAGS.logging_steps,
+        save_steps=FLAGS.save_steps,
+        lr_scheduler_type=FLAGS.schedule,
+        optim=FLAGS.optim,
+        weight_decay=FLAGS.weight_decay,
+        warmup_steps=FLAGS.warmup_steps,
+        fsdp=FLAGS.fsdp,
+        fsdp_config=FLAGS.fsdp_config if FLAGS.fsdp != '' else None,
+        push_to_hub=FLAGS.push_to_hub,
+        num_train_epochs=num_train_epochs,
+        hub_model_id=FLAGS.hub_model_id
+    )
+
+    flags_dict = FLAGS.flag_values_dict()
+    trainer = SFTTrainer(
+        model=model,
+        args=transformer_args,
+        tokenizer=tokenizer,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+        formatting_func=formatting_prompts_func,
+        data_collator=collator,
+        dataset_batch_size=FLAGS.batch_size,
+    )
+
+    trainer.train()
+
+if __name__ == '__main__':
+    app.run(main)

wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/conda-environment.yaml

@@ -0,0 +1,515 @@
+name: base
+channels:
+  - fastai
+  - nvidia
+  - https://aws-ml-conda.s3.us-west-2.amazonaws.com
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_kmp_llvm
+  - alsa-lib=1.2.12=h4ab18f5_0
+  - annotated-types=0.7.0=pyhd8ed1ab_0
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - attr=2.5.1=h166bdaf_1
+  - aws-ofi-nccl=1.9.1=aws_efa1.26.1_0
+  - blas=1.0=mkl
+  - boltons=23.0.0=pyhd8ed1ab_0
+  - brotli-bin=1.1.0=hd590300_0
+  - bzip2=1.0.8=h7f98852_4
+  - c-ares=1.32.3=h4bc722e_0
+  - ca-certificates=2024.7.4=hbcca054_0
+  - cached-property=1.5.2=hd8ed1ab_1
+  - cached_property=1.5.2=pyha770c72_1
+  - cairo=1.18.0=h3faef2a_0
+  - catalogue=2.0.10=py310hff52083_0
+  - certifi=2024.7.4=pyhd8ed1ab_0
+  - cffi=1.15.1=py310h255011f_3
+  - charset-normalizer=3.2.0=pyhd8ed1ab_0
+  - click=8.1.7=unix_pyh707e725_0
+  - cloudpathlib=0.18.1=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - comm=0.2.2=pyhd8ed1ab_0
+  - conda=23.3.1=py310hff52083_0
+  - conda-libmamba-solver=23.3.0=pyhd8ed1ab_0
+  - conda-package-handling=2.2.0=pyh38be061_0
+  - conda-package-streaming=0.9.0=pyhd8ed1ab_0
+  - confection=0.1.4=py310h17c5347_0
+  - contourpy=1.2.1=py310hd41b1e2_0
+  - cuda-cccl=12.4.127=0
+  - cuda-compiler=12.1.1=0
+  - cuda-cudart=12.1.105=0
+  - cuda-cudart-dev=12.1.105=0
+  - cuda-cudart-static=12.1.105=0
+  - cuda-cuobjdump=12.4.127=0
+  - cuda-cupti=12.1.105=0
+  - cuda-cupti-static=12.1.105=0
+  - cuda-cuxxfilt=12.4.127=0
+  - cuda-driver-dev=12.4.127=0
+  - cuda-libraries=12.1.0=0
+  - cuda-libraries-dev=12.1.0=0
+  - cuda-libraries-static=12.1.1=0
+  - cuda-nvcc=12.1.105=0
+  - cuda-nvprune=12.4.127=0
+  - cuda-nvrtc=12.1.105=0
+  - cuda-nvrtc-dev=12.1.105=0
+  - cuda-nvrtc-static=12.1.105=0
+  - cuda-nvtx=12.1.105=0
+  - cuda-opencl=12.4.127=0
+  - cuda-opencl-dev=12.4.127=0
+  - cuda-profiler-api=12.4.127=0
+  - cuda-runtime=12.1.0=0
+  - cuda-version=12.1=h1d6eff3_3
+  - curl=8.8.0=he654da7_1
+  - cycler=0.12.1=pyhd8ed1ab_0
+  - cymem=2.0.8=py310hc6cd4ac_1
+  - cython=3.0.11=py310hea249c9_0
+  - cython-blis=0.7.10=py310h1f7b6fc_2
+  - dbus=1.13.6=h5008d03_3
+  - debugpy=1.8.5=py310hea249c9_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - docutils=0.16=py310hff52083_4
+  - exceptiongroup=1.2.2=pyhd8ed1ab_0
+  - executing=2.0.1=pyhd8ed1ab_0
+  - expat=2.6.2=h59595ed_0
+  - fastai=2.7.16=py_0
+  - fastcore=1.5.55=py_0
+  - fastdownload=0.0.7=py_0
+  - fastprogress=1.0.3=py_0
+  - ffmpeg=4.2=h3fd9d12_1
+  - filelock=3.15.4=pyhd8ed1ab_0
+  - fmt=9.1.0=h924138e_0
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=h77eed37_2
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - fonttools=4.53.1=py310h5b4e0ec_0
+  - freetype=2.12.1=h267a509_2
+  - gettext=0.22.5=h59595ed_2
+  - gettext-tools=0.22.5=h59595ed_2
+  - glib=2.80.2=hf974151_0
+  - glib-tools=2.80.2=hb6ce0ca_0
+  - gmp=6.3.0=hac33072_2
+  - gmpy2=2.1.5=py310hc7909c9_1
+  - gnutls=3.6.15=he1e5248_0
+  - graphite2=1.3.13=h59595ed_1003
+  - gst-plugins-base=1.22.9=hfa15dee_1
+  - gstreamer=1.22.9=h98fc4e7_1
+  - h5py=3.11.0=nompi_py310hf054cd7_102
+  - harfbuzz=8.5.0=hfac3d4d_0
+  - hdf5=1.14.3=nompi_hdf9ad27_105
+  - hwloc=2.9.2=h2bc3f7f_0
+  - icu=73.2=h59595ed_0
+  - idna=3.7=pyhd8ed1ab_0
+  - importlib_metadata=8.2.0=hd8ed1ab_0
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.4=pyhd8ed1ab_0
+  - jmespath=1.0.1=pyhd8ed1ab_0
+  - joblib=1.4.2=pyhd8ed1ab_0
+  - jsonpatch=1.32=pyhd8ed1ab_0
+  - jsonpointer=2.0=py_0
+  - jupyter_client=8.6.2=pyhd8ed1ab_0
+  - jupyter_core=5.7.2=py310hff52083_0
+  - keyutils=1.6.1=h166bdaf_0
+  - kiwisolver=1.4.5=py310hd41b1e2_1
+  - krb5=1.21.3=h659f571_0
+  - lame=3.100=h166bdaf_1003
+  - langcodes=3.4.0=pyhd8ed1ab_0
+  - language-data=1.2.0=pyhd8ed1ab_0
+  - lcms2=2.16=hb7c19ff_0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - lerc=4.0.0=h27087fc_0
+  - libaec=1.1.3=h59595ed_0
+  - libarchive=3.6.2=h039dbb9_1
+  - libasprintf=0.22.5=h661eb56_2
+  - libasprintf-devel=0.22.5=h661eb56_2
+  - libblas=3.9.0=21_linux64_mkl
+  - libbrotlicommon=1.1.0=hd590300_0
+  - libbrotlidec=1.1.0=hd590300_0
+  - libbrotlienc=1.1.0=hd590300_0
+  - libcap=2.69=h0f662aa_0
+  - libcblas=3.9.0=21_linux64_mkl
+  - libclang=15.0.7=default_h127d8a8_5
+  - libclang13=15.0.7=default_h5d6823c_5
+  - libcublas=12.1.0.26=0
+  - libcublas-dev=12.1.0.26=0
+  - libcublas-static=12.1.3.1=hd3aeb46_0
+  - libcufft=11.0.2.4=0
+  - libcufft-dev=11.0.2.4=0
+  - libcufft-static=11.0.2.54=hd3aeb46_0
+  - libcufile=1.9.1.3=0
+  - libcufile-dev=1.9.1.3=0
+  - libcufile-static=1.9.1.3=0
+  - libcups=2.3.3=h4637d8d_4
+  - libcurand=10.3.5.147=0
+  - libcurand-dev=10.3.5.147=0
+  - libcurand-static=10.3.5.147=0
+  - libcurl=8.8.0=hca28451_1
+  - libcusolver=11.4.4.55=0
+  - libcusolver-dev=11.4.4.55=0
+  - libcusolver-static=11.4.5.107=hd3aeb46_0
+  - libcusparse=12.0.2.55=0
+  - libcusparse-dev=12.0.2.55=0
+  - libcusparse-static=12.1.0.106=hd3aeb46_0
+  - libdeflate=1.20=hd590300_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=h516909a_1
+  - libevent=2.1.12=hf998b51_1
+  - libexpat=2.6.2=h59595ed_0
+  - libffi=3.4.2=h7f98852_5
+  - libflac=1.4.3=h59595ed_0
+  - libgcc=7.2.0=h69d50b8_2
+  - libgcc-ng=13.1.0=he5830b7_0
+  - libgcrypt=1.11.0=h4ab18f5_1
+  - libgettextpo=0.22.5=h59595ed_2
+  - libgettextpo-devel=0.22.5=h59595ed_2
+  - libgfortran-ng=13.2.0=h69a702a_0
+  - libgfortran5=13.2.0=ha4646dd_0
+  - libglib=2.80.2=hf974151_0
+  - libgomp=13.1.0=he5830b7_0
+  - libgpg-error=1.50=h4f305b6_0
+  - libhwloc=2.9.3=default_h554bfaf_1009
+  - libiconv=1.17=h166bdaf_0
+  - libidn2=2.3.7=hd590300_0
+  - libjpeg-turbo=3.0.0=hd590300_1
+  - liblapack=3.9.0=21_linux64_mkl
+  - libllvm15=15.0.7=h5cf9203_3
+  - libmamba=1.4.2=hcea66bb_0
+  - libmambapy=1.4.2=py310h1428755_0
+  - libnghttp2=1.58.0=h47da74e_1
+  - libnl=3.10.0=h4bc722e_0
+  - libnpp=12.0.2.50=0
+  - libnpp-dev=12.0.2.50=0
+  - libnpp-static=12.1.0.40=hd3aeb46_0
+  - libnsl=2.0.0=h7f98852_0
+  - libnvjitlink=12.1.105=0
+  - libnvjitlink-dev=12.1.105=0
+  - libnvjitlink-static=12.1.105=hd3aeb46_0
+  - libnvjpeg=12.1.1.14=0
+  - libnvjpeg-dev=12.1.1.14=0
+  - libnvjpeg-static=12.2.0.2=ha770c72_0
+  - libogg=1.3.5=h4ab18f5_0
+  - libopenblas=0.3.27=pthreads_hac2b453_1
+  - libopus=1.3.1=h7f98852_1
+  - libpng=1.6.43=h2797004_0
+  - libpq=16.4=h482b261_0
+  - libsndfile=1.2.2=hc60ed4a_1
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsolv=0.7.29=ha6fb4c9_0
+  - libsqlite=3.46.0=hde9e2c9_0
+  - libssh2=1.11.0=h0841786_0
+  - libstdcxx-ng=13.1.0=hfd8a6a1_0
+  - libsystemd0=255=h3516f8a_1
+  - libtasn1=4.19.0=h166bdaf_0
+  - libtiff=4.6.0=h1dd3fc0_3
+  - libunistring=0.9.10=h7f98852_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libuv=1.48.0=hd590300_0
+  - libvorbis=1.3.7=h9c3ff4c_0
+  - libwebp-base=1.4.0=hd590300_0
+  - libxcb=1.15=h0b41bf4_0
+  - libxkbcommon=1.6.0=h5d7e998_0
+  - libxml2=2.11.6=h232c23b_0
+  - libzlib=1.2.13=h4ab18f5_6
+  - llvm-openmp=18.1.7=ha31de31_0
+  - lz4-c=1.9.4=hcb278e6_0
+  - lzo=2.10=h516909a_1000
+  - mamba=1.4.2=py310h51d5547_0
+  - marisa-trie=1.1.0=py310hc6cd4ac_1
+  - markdown-it-py=3.0.0=pyhd8ed1ab_0
+  - markupsafe=2.1.5=py310h2372a71_0
+  - matplotlib=3.9.1=py310hff52083_1
+  - matplotlib-base=3.9.1=py310hf02ac8c_2
+  - matplotlib-inline=0.1.7=pyhd8ed1ab_0
+  - mdurl=0.1.2=pyhd8ed1ab_0
+  - mkl=2024.0.0=ha957f24_49657
+  - mkl-include=2024.2.0=ha957f24_665
+  - mpc=1.3.1=hfe3b2da_0
+  - mpfr=4.2.1=h38ae2d0_2
+  - mpg123=1.32.6=h59595ed_0
+  - mpi=1.0=openmpi
+  - mpi4py=3.1.6=py310hb2ba3f8_1
+  - mpmath=1.3.0=pyhd8ed1ab_0
+  - munkres=1.1.4=pyh9f0ad1d_0
+  - murmurhash=1.0.10=py310hc6cd4ac_1
+  - mysql-common=8.0.33=hf1915f5_6
+  - mysql-libs=8.0.33=hca2cd23_6
+  - ncurses=6.5=h59595ed_0
+  - nest-asyncio=1.6.0=pyhd8ed1ab_0
+  - nettle=3.7.3=hbbd107a_1
+  - networkx=3.3=pyhd8ed1ab_1
+  - nspr=4.35=h27087fc_0
+  - nss=3.100=hca3bf56_0
+  - openh264=2.1.1=h780b84a_0
+  - openjpeg=2.5.2=h488ebb8_0
+  - openmpi=5.0.3=h817cd4e_101
+  - openssl=3.3.1=h4bc722e_2
+  - packaging=23.1=pyhd8ed1ab_0
+  - pandas=2.2.2=py310hf9f9076_1
+  - parso=0.8.4=pyhd8ed1ab_0
+  - pcre2=10.43=hcad00b1_0
+  - pexpect=4.9.0=pyhd8ed1ab_0
+  - pickleshare=0.7.5=py_1003
+  - pillow=10.3.0=py310hf73ecf8_0
+  - pixman=0.43.2=h59595ed_0
+  - platformdirs=4.2.2=pyhd8ed1ab_0
+  - pluggy=1.3.0=pyhd8ed1ab_0
+  - ply=3.11=pyhd8ed1ab_2
+  - preshed=3.0.9=py310hc6cd4ac_1
+  - prompt-toolkit=3.0.47=pyha770c72_0
+  - psutil=6.0.0=py310hc51659f_0
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pulseaudio-client=16.1=hb77b528_5
+  - pure_eval=0.2.3=pyhd8ed1ab_0
+  - pyasn1=0.6.0=pyhd8ed1ab_0
+  - pybind11=2.13.1=py310h25c7140_0
+  - pybind11-abi=4=hd8ed1ab_3
+  - pybind11-global=2.13.1=py310h25c7140_0
+  - pycosat=0.6.4=py310h5764c6d_1
+  - pycparser=2.21=pyhd8ed1ab_0
+  - pydantic=2.8.2=pyhd8ed1ab_0
+  - pydantic-core=2.20.1=py310h42e942d_0
+  - pygments=2.18.0=pyhd8ed1ab_0
+  - pyparsing=3.1.2=pyhd8ed1ab_0
+  - pyqt=5.15.9=py310h04931ad_5
+  - pyqt5-sip=12.12.2=py310hc6cd4ac_5
+  - pysocks=1.7.1=pyha2e5f31_6
+  - python=3.10.12=hd12c33a_0_cpython
+  - python-dateutil=2.9.0=pyhd8ed1ab_0
+  - python-tzdata=2024.1=pyhd8ed1ab_0
+  - python_abi=3.10=3_cp310
+  - pytorch-cuda=12.1=ha16c6d3_5
+  - pytorch-mutex=1.0=cuda
+  - pytz=2024.1=pyhd8ed1ab_0
+  - pyyaml=6.0.2=py310h5b4e0ec_0
+  - pyzmq=26.1.0=py310h7d2b5bf_0
+  - qhull=2020.2=h434a139_5
+  - qt-main=5.15.8=h82b777d_17
+  - readline=8.2=h8228510_1
+  - reproc=14.2.4=h0b41bf4_0
+  - reproc-cpp=14.2.4=hcb278e6_0
+  - requests=2.32.3=pyhd8ed1ab_0
+  - rhash=1.4.4=hd590300_0
+  - rich=13.7.1=pyhd8ed1ab_0
+  - rsa=4.7.2=pyh44b312d_0
+  - ruamel.yaml=0.17.32=py310h2372a71_0
+  - ruamel.yaml.clib=0.2.7=py310h1fa729e_1
+  - s3transfer=0.10.2=pyhd8ed1ab_0
+  - scipy=1.14.0=py310h93e2701_1
+  - shellingham=1.5.4=pyhd8ed1ab_0
+  - sip=6.7.12=py310hc6cd4ac_0
+  - six=1.16.0=pyh6c4a22f_0
+  - smart-open=7.0.4=hd8ed1ab_0
+  - smart_open=7.0.4=pyhd8ed1ab_0
+  - spacy=3.7.5=py310h9fd543b_1
+  - spacy-legacy=3.0.12=pyhd8ed1ab_0
+  - spacy-loggers=1.0.5=pyhd8ed1ab_0
+  - srsly=2.4.8=py310hc6cd4ac_1
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - sympy=1.13.0=pypyh2585a3b_103
+  - tbb=2021.8.0=hdb19cb5_0
+  - thinc=8.2.5=py310h9fd543b_0
+  - threadpoolctl=3.5.0=pyhc1e730c_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - toml=0.10.2=pyhd8ed1ab_0
+  - tomli=2.0.1=pyhd8ed1ab_0
+  - toolz=0.12.0=pyhd8ed1ab_0
+  - torchdata=0.7.0=py310
+  - torchtext=0.16.0=py310
+  - tornado=6.4.1=py310hc51659f_0
+  - tqdm=4.66.5=pyhd8ed1ab_0
+  - traitlets=5.14.3=pyhd8ed1ab_0
+  - typer=0.12.3=pyhd8ed1ab_0
+  - typer-slim=0.12.3=pyhd8ed1ab_0
+  - typer-slim-standard=0.12.3=hd8ed1ab_0
+  - typing=3.10.0.0=pyhd8ed1ab_1
+  - typing-extensions=4.12.2=hd8ed1ab_0
+  - typing_extensions=4.12.2=pyha770c72_0
+  - tzdata=2023c=h71feb2d_0
+  - unicodedata2=15.1.0=py310h2372a71_0
+  - urllib3=1.26.19=pyhd8ed1ab_0
+  - wasabi=1.1.2=py310hff52083_1
+  - wcwidth=0.2.13=pyhd8ed1ab_0
+  - weasel=0.4.1=pyhd8ed1ab_1
+  - wheel=0.41.2=pyhd8ed1ab_0
+  - wrapt=1.16.0=py310h2372a71_0
+  - xcb-util=0.4.0=hd590300_1
+  - xcb-util-image=0.4.0=h8ee46fc_1
+  - xcb-util-keysyms=0.4.0=h8ee46fc_1
+  - xcb-util-renderutil=0.3.9=hd590300_1
+  - xcb-util-wm=0.4.1=h8ee46fc_1
+  - xkeyboard-config=2.42=h4ab18f5_0
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.9=h8ee46fc_0
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xf86vidmodeproto=2.3.1=h7f98852_1002
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xz=5.2.6=h166bdaf_0
+  - yaml=0.2.5=h7f98852_2
+  - yaml-cpp=0.7.0=h27087fc_2
+  - zeromq=4.3.5=h75354e8_4
+  - zipp=3.19.2=pyhd8ed1ab_0
+  - zlib=1.2.13=h4ab18f5_6
+  - zstandard=0.19.0=py310h5764c6d_0
+  - zstd=1.5.6=ha6fb4c9_0
+  - pip:
+      - absl-py==2.1.0
+      - accelerate==1.1.1
+      - aiohappyeyeballs==2.4.3
+      - aiohttp==3.10.10
+      - aiosignal==1.3.1
+      - antlr4-python3-runtime==4.9.3
+      - apex==0.1
+      - appdirs==1.4.4
+      - argparse==1.4.0
+      - async-timeout==4.0.3
+      - attrdict==2.0.1
+      - attrs==23.2.0
+      - awscli==1.33.40
+      - backcall==0.2.0
+      - bcrypt==4.2.0
+      - beautifulsoup4==4.12.3
+      - bleach==6.2.0
+      - bokeh==3.5.1
+      - boto3==1.34.158
+      - botocore==1.34.158
+      - brotli==1.0.9
+      - cachetools==5.5.0
+      - chardet==5.2.0
+      - cloudpickle==2.2.1
+      - cmake==3.30.2
+      - cryptography==41.0.7
+      - datasets==3.1.0
+      - defusedxml==0.7.1
+      - dill==0.3.8
+      - docker==7.1.0
+      - docker-pycreds==0.4.0
+      - docopt==0.6.2
+      - docstring-parser==0.16
+      - einops==0.8.0
+      - fastjsonschema==2.20.0
+      - fire==0.6.0
+      - flash-attn==2.5.5
+      - frozenlist==1.5.0
+      - fsspec==2024.6.1
+      - gevent==24.2.1
+      - gitdb==4.0.11
+      - gitpython==3.1.43
+      - google-pasta==0.2.0
+      - greenlet==3.0.3
+      - huggingface-hub==0.26.2
+      - hydra-core==1.3.2
+      - imageio==2.34.2
+      - importlib-metadata==6.11.0
+      - importlib-resources==6.1.1
+      - inotify-simple==1.2.1
+      - ipykernel==6.29.1
+      - ipython==8.12.3
+      - jsonschema==4.23.0
+      - jsonschema-specifications==2023.12.1
+      - jupyterlab-pygments==0.3.0
+      - llvmlite==0.43.0
+      - mistune==3.0.2
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - nbclient==0.10.0
+      - nbconvert==7.16.4
+      - nbformat==5.10.4
+      - ninja==1.11.1.1
+      - numba==0.60.0
+      - numpy==1.24.3
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==8.9.2.26
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-ml-py==12.535.161
+      - nvidia-nccl-cu12==2.19.3
+      - nvidia-nvjitlink-cu12==12.6.77
+      - nvidia-nvtx-cu12==12.1.105
+      - nvitop==1.3.2
+      - omegaconf==2.3.0
+      - opencv-python==4.10.0.84
+      - pandocfilters==1.5.1
+      - paramiko==3.4.0
+      - pathos==0.3.2
+      - pip==24.1.2
+      - pip-chill==1.0.3
+      - pipreqs==0.5.0
+      - plotly==5.23.0
+      - pox==0.3.4
+      - ppft==1.7.6.8
+      - propcache==0.2.0
+      - protobuf==3.20.3
+      - pyarrow==17.0.0
+      - pyfunctional==1.5.0
+      - pynacl==1.5.0
+      - pyopenssl==23.2.0
+      - referencing==0.35.1
+      - regex==2024.11.6
+      - retrying==1.3.4
+      - rpds-py==0.20.0
+      - s3fs==0.4.2
+      - safetensors==0.4.5
+      - sagemaker==2.224.2
+      - sagemaker-experiments==0.1.45
+      - sagemaker-pytorch-training==2.8.1
+      - sagemaker-training==4.8.1
+      - schema==0.7.7
+      - scikit-learn==1.3.2
+      - seaborn==0.13.2
+      - sentencepiece==0.2.0
+      - sentry-sdk==2.18.0
+      - setproctitle==1.3.3
+      - setuptools==72.1.0
+      - shap==0.46.0
+      - shtab==1.7.1
+      - slicer==0.0.8
+      - smclarify==0.5
+      - smdebug-rulesconfig==1.0.1
+      - smdistributed-dataparallel==2.1.0
+      - smmap==5.0.1
+      - smprof==0.3.334
+      - soupsieve==2.6
+      - tabulate==0.9.0
+      - tblib==3.0.0
+      - tenacity==9.0.0
+      - tensor-parallel==2.0.0
+      - termcolor==2.5.0
+      - tinycss2==1.4.0
+      - tokenizers==0.20.3
+      - torch==2.2.0
+      - torchaudio==2.2.0
+      - torchnet==0.0.4
+      - torchvision==0.17.0
+      - transformer-engine==0.12.0+170797
+      - transformers==4.46.0
+      - triton==2.2.0
+      - trl==0.12.0
+      - tyro==0.8.5
+      - visdom==0.2.4
+      - wandb==0.16.3
+      - webencodings==0.5.1
+      - websocket-client==1.8.0
+      - werkzeug==3.0.3
+      - xxhash==3.5.0
+      - xyzservices==2024.6.0
+      - yarg==0.1.9
+      - yarl==1.17.1
+      - zope-event==5.0
+      - zope-interface==7.0.1
+prefix: /opt/conda

wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/config.yaml

@@ -0,0 +1,948 @@
+wandb_version: 1
+
+sagemaker_training_job_name:
+  desc: null
+  value: kushalarora-rvv-main-2024-11-13-17-55-42-664
+batch_size:
+  desc: null
+  value: 64
+cache_dir:
+  desc: null
+  value: /opt/ml/data/input/.cache
+dataset:
+  desc: null
+  value: code-contests
+gradient_accumulation_steps:
+  desc: null
+  value: 8
+logging_steps:
+  desc: null
+  value: 5
+lr:
+  desc: null
+  value: 1e-6
+max_seq_length:
+  desc: null
+  value: 2048
+model_name:
+  desc: null
+  value: Qwen/Qwen2.5-Coder-7B-Instruct
+num_train_epochs:
+  desc: null
+  value: 2
+output_dir:
+  desc: null
+  value: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048
+push_to_hub:
+  desc: null
+  value: true
+run_name:
+  desc: null
+  value: sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048
+sagemaker_container_log_level:
+  desc: null
+  value: 20
+sagemaker_instance_type:
+  desc: null
+  value: ml.p5.48xlarge
+sagemaker_job_name:
+  desc: null
+  value: kushalarora-rvv-main-2024-11-13-17-55-42-664
+sagemaker_program:
+  desc: null
+  value: sft.py
+sagemaker_region:
+  desc: null
+  value: us-east-1
+sagemaker_submit_directory:
+  desc: null
+  value: s3://tri-ml-datasets/kushalarora-rvv-main-2024-11-13-17-55-42-664/source/sourcedir.tar.gz
+sagemaker_torch_distributed_enabled:
+  desc: null
+  value: 'true'
+save_steps:
+  desc: null
+  value: 100
+schedule:
+  desc: null
+  value: cosine
+wandb_project:
+  desc: null
+  value: sft-codecontests-1112
+weight_decay:
+  desc: null
+  value: 0.0
+additional_framework_parameters:
+  desc: null
+  value:
+    sagemaker_instance_type: ml.p5.48xlarge
+    sagemaker_torch_distributed_enabled: true
+channel_input_dirs:
+  desc: null
+  value: {}
+current_host:
+  desc: null
+  value: algo-1
+current_instance_group:
+  desc: null
+  value: homogeneousCluster
+current_instance_group_hosts:
+  desc: null
+  value:
+  - algo-1
+current_instance_type:
+  desc: null
+  value: ml.p5.48xlarge
+distribution_hosts:
+  desc: null
+  value:
+  - algo-1
+distribution_instance_groups:
+  desc: null
+  value:
+  - homogeneousCluster
+framework_module:
+  desc: null
+  value: sagemaker_pytorch_container.training:main
+hosts:
+  desc: null
+  value:
+  - algo-1
+hyperparameters:
+  desc: null
+  value:
+    batch_size: 64
+    cache_dir: /opt/ml/data/input/.cache
+    dataset: code-contests
+    gradient_accumulation_steps: 8
+    logging_steps: 5
+    lr: 1e-6
+    max_seq_length: 2048
+    model_name: Qwen/Qwen2.5-Coder-7B-Instruct
+    num_train_epochs: 2
+    output_dir: /opt/ml/model/
+    push_to_hub: true
+    run_name: sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048
+    save_steps: 100
+    schedule: cosine
+    wandb_project: sft-codecontests-1112
+    weight_decay: 0.0
+input_config_dir:
+  desc: null
+  value: /opt/ml/input/config
+input_data_config:
+  desc: null
+  value: {}
+input_dir:
+  desc: null
+  value: /opt/ml/input
+instance_groups:
+  desc: null
+  value:
+  - homogeneousCluster
+instance_groups_dict:
+  desc: null
+  value:
+    homogeneousCluster:
+      hosts:
+      - algo-1
+      instance_group_name: homogeneousCluster
+      instance_type: ml.p5.48xlarge
+is_hetero:
+  desc: null
+  value: false
+is_master:
+  desc: null
+  value: true
+is_modelparallel_enabled:
+  desc: null
+  value: null
+is_smddpmprun_installed:
+  desc: null
+  value: false
+is_smddprun_installed:
+  desc: null
+  value: true
+job_name:
+  desc: null
+  value: kushalarora-rvv-main-2024-11-13-17-55-42-664
+log_level:
+  desc: null
+  value: passive
+master_hostname:
+  desc: null
+  value: algo-1
+model_dir:
+  desc: null
+  value: /opt/ml/model
+module_dir:
+  desc: null
+  value: s3://tri-ml-datasets/kushalarora-rvv-main-2024-11-13-17-55-42-664/source/sourcedir.tar.gz
+module_name:
+  desc: null
+  value: /opt/ml/code/qlearning_reasoning/training/sft
+network_interface_name:
+  desc: null
+  value: eth0
+num_cpus:
+  desc: null
+  value: 192
+num_gpus:
+  desc: null
+  value: 8
+num_neurons:
+  desc: null
+  value: 0
+output_data_dir:
+  desc: null
+  value: /opt/ml/output/data
+output_intermediate_dir:
+  desc: null
+  value: /opt/ml/output/intermediate
+resource_config:
+  desc: null
+  value:
+    current_group_name: homogeneousCluster
+    current_host: algo-1
+    current_instance_type: ml.p5.48xlarge
+    hosts:
+    - algo-1
+    instance_groups:
+    - hosts:
+      - algo-1
+      instance_group_name: homogeneousCluster
+      instance_type: ml.p5.48xlarge
+    network_interface_name: eth0
+user_entry_point:
+  desc: null
+  value: /opt/ml/code/qlearning_reasoning/training/sft.py
+_wandb:
+  desc: null
+  value:
+    code_path: code/qlearning_reasoning/training/sft.py
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.46.0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1731524566.431221
+    t:
+      1:
+      - 1
+      - 5
+      - 11
+      - 41
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 84
+      2:
+      - 1
+      - 5
+      - 11
+      - 41
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 84
+      3:
+      - 7
+      - 9
+      - 13
+      - 19
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.46.0
+      8:
+      - 5
+      9:
+        1: transformers_trainer
+      13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+    - 1: train/loss
+      5: 1
+      6:
+      - 1
+    - 1: train/grad_norm
+      5: 1
+      6:
+      - 1
+    - 1: train/learning_rate
+      5: 1
+      6:
+      - 1
+    - 1: train/epoch
+      5: 1
+      6:
+      - 1
+    - 1: eval/loss
+      5: 1
+      6:
+      - 1
+    - 1: eval/runtime
+      5: 1
+      6:
+      - 1
+    - 1: eval/samples_per_second
+      5: 1
+      6:
+      - 1
+    - 1: eval/steps_per_second
+      5: 1
+      6:
+      - 1
+vocab_size:
+  desc: null
+  value: 151671
+max_position_embeddings:
+  desc: null
+  value: 32768
+hidden_size:
+  desc: null
+  value: 3584
+intermediate_size:
+  desc: null
+  value: 18944
+num_hidden_layers:
+  desc: null
+  value: 28
+num_attention_heads:
+  desc: null
+  value: 28
+use_sliding_window:
+  desc: null
+  value: false
+sliding_window:
+  desc: null
+  value: null
+max_window_layers:
+  desc: null
+  value: 28
+num_key_value_heads:
+  desc: null
+  value: 4
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-06
+use_cache:
+  desc: null
+  value: false
+rope_theta:
+  desc: null
+  value: 1000000.0
+rope_scaling:
+  desc: null
+  value: null
+attention_dropout:
+  desc: null
+  value: 0.0
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: bfloat16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - Qwen2ForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: null
+pad_token_id:
+  desc: null
+  value: 151643
+eos_token_id:
+  desc: null
+  value: 151645
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: Qwen/Qwen2.5-Coder-7B-Instruct
+_attn_implementation_autoset:
+  desc: null
+  value: true
+transformers_version:
+  desc: null
+  value: 4.46.0
+model_type:
+  desc: null
+  value: qwen2
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: true
+do_predict:
+  desc: null
+  value: false
+eval_strategy:
+  desc: null
+  value: steps
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 8
+per_device_eval_batch_size:
+  desc: null
+  value: 8
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_delay:
+  desc: null
+  value: 0
+torch_empty_cache_steps:
+  desc: null
+  value: null
+learning_rate:
+  desc: null
+  value: 1.0e-06
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: cosine
+lr_scheduler_kwargs:
+  desc: null
+  value: {}
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 0
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/runs/Nov13_19-02-15_algo-1
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: true
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: epoch
+save_total_limit:
+  desc: null
+  value: null
+save_safetensors:
+  desc: null
+  value: true
+save_on_each_node:
+  desc: null
+  value: false
+save_only_model:
+  desc: null
+  value: false
+restore_callback_states_from_checkpoint:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_cpu:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: null
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: false
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: null
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: null
+tpu_num_cores:
+  desc: null
+  value: null
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: []
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: 5
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_prefetch_factor:
+  desc: null
+  value: null
+past_index:
+  desc: null
+  value: -1
+disable_tqdm:
+  desc: null
+  value: false
+remove_unused_columns:
+  desc: null
+  value: true
+label_names:
+  desc: null
+  value: null
+load_best_model_at_end:
+  desc: null
+  value: false
+metric_for_best_model:
+  desc: null
+  value: null
+greater_is_better:
+  desc: null
+  value: null
+ignore_data_skip:
+  desc: null
+  value: false
+fsdp:
+  desc: null
+  value:
+  - full_shard
+  - auto_wrap
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value:
+    activation_checkpointing: false
+    auto_wrap_policy: TRANSFORMER_BASED_WRAP
+    backward_prefetch: BACKWARD_PRE
+    cpu_ram_efficient_loading: true
+    forward_prefetch: false
+    offload_params: false
+    sharding_strategy: FULL_SHARD
+    state_dict_type: FULL_STATE_DICT
+    sync_module_states: true
+    min_num_params: 0
+    xla: false
+    xla_fsdp_v2: false
+    xla_fsdp_grad_ckpt: false
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: null
+accelerator_config:
+  desc: null
+  value:
+    split_batches: false
+    dispatch_batches: null
+    even_batches: true
+    use_seedable_sampler: true
+    non_blocking: false
+    gradient_accumulation_kwargs: null
+deepspeed:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: adamw_torch
+optim_args:
+  desc: null
+  value: null
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value:
+  - wandb
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+ddp_bucket_cap_mb:
+  desc: null
+  value: null
+ddp_broadcast_buffers:
+  desc: null
+  value: null
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataloader_persistent_workers:
+  desc: null
+  value: false
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+resume_from_checkpoint:
+  desc: null
+  value: null
+hub_model_id:
+  desc: null
+  value: sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+hub_always_push:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: false
+gradient_checkpointing_kwargs:
+  desc: null
+  value:
+    use_reentrant: false
+include_inputs_for_metrics:
+  desc: null
+  value: false
+include_for_metrics:
+  desc: null
+  value: []
+eval_do_concat_batches:
+  desc: null
+  value: true
+fp16_backend:
+  desc: null
+  value: auto
+evaluation_strategy:
+  desc: null
+  value: null
+push_to_hub_model_id:
+  desc: null
+  value: null
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: null
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: null
+torch_compile_mode:
+  desc: null
+  value: null
+dispatch_batches:
+  desc: null
+  value: null
+split_batches:
+  desc: null
+  value: null
+include_tokens_per_second:
+  desc: null
+  value: false
+include_num_input_tokens_seen:
+  desc: null
+  value: false
+neftune_noise_alpha:
+  desc: null
+  value: null
+optim_target_modules:
+  desc: null
+  value: null
+batch_eval_metrics:
+  desc: null
+  value: false
+eval_on_start:
+  desc: null
+  value: false
+use_liger_kernel:
+  desc: null
+  value: false
+eval_use_gather_object:
+  desc: null
+  value: false
+dataset_text_field:
+  desc: null
+  value: text
+packing:
+  desc: null
+  value: false
+dataset_num_proc:
+  desc: null
+  value: null
+dataset_batch_size:
+  desc: null
+  value: 64
+model_init_kwargs:
+  desc: null
+  value: null
+dataset_kwargs:
+  desc: null
+  value: {}
+eval_packing:
+  desc: null
+  value: null
+num_of_sequences:
+  desc: null
+  value: 1024
+chars_per_token:
+  desc: null
+  value: <CHARS_PER_TOKEN>
+use_liger:
+  desc: null
+  value: false
+model/num_parameters:
+  desc: null
+  value: 951599936

wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/diff.patch

@@ -0,0 +1,49 @@
+diff --git a/qlearning_reasoning/training/sft.py b/qlearning_reasoning/training/sft.py
+index 583fba2..1c2b359 100644
+--- a/qlearning_reasoning/training/sft.py
++++ b/qlearning_reasoning/training/sft.py
+@@ -51,6 +51,7 @@ flags.DEFINE_string('fsdp', "full_shard auto_wrap", 'If to use FSDP')
+ flags.DEFINE_string('fsdp_config', "qlearning_reasoning/accelerate_configs/fsdp_config.json", 'FSDP config to use.')
+ flags.DEFINE_boolean("push_to_hub", False, "Should the model be uploaded to Huggingface hub?")
+ flags.DEFINE_boolean('add_step_tokens', True, 'Add step tokens')
++flags.DEFINE_string("hub_model_id", None, "Hub model id. If not defined it will be truncated run name.")
+ 
+ START_OF_STEP_TOKEN = '[STEP]'
+ END_OF_STEP_TOKEN = '[/STEP]'
+@@ -60,6 +61,10 @@ START_OF_REVISION_TOKEN = '[REVISION]'
+ END_OF_REVISION_TOKEN = '[/REVISION]'
+ 
+ def main(_):
++
++    if FLAGS.hub_model_id is None:
++        FLAGS.hub_model_id = FLAGS.run_name[:96]
++
+     if not os.path.exists(FLAGS.output_dir):
+         os.makedirs(FLAGS.output_dir, exist_ok=True)
+         os.makedirs(f"{FLAGS.output_dir}/{FLAGS.run_name}", exist_ok=True)
+@@ -219,11 +224,11 @@ def main(_):
+         optim=FLAGS.optim,
+         weight_decay=FLAGS.weight_decay,
+         warmup_steps=FLAGS.warmup_steps,
+-        max_steps=num_steps,
+         fsdp=FLAGS.fsdp,
+         fsdp_config=FLAGS.fsdp_config if FLAGS.fsdp != '' else None,
+         push_to_hub=FLAGS.push_to_hub,
+         num_train_epochs=num_train_epochs,
++        hub_model_id=FLAGS.hub_model_id
+     )
+ 
+     flags_dict = FLAGS.flag_values_dict()
+diff --git a/sagemaker/launch_sagemaker.py b/sagemaker/launch_sagemaker.py
+index 3316ede..98ed205 100644
+--- a/sagemaker/launch_sagemaker.py
++++ b/sagemaker/launch_sagemaker.py
+@@ -224,7 +224,7 @@ def main(argv):
+     # TODO: verify if this breaks the code
+     environment = {
+         "PYTHONPATH": "/opt/ml/code/qlearning_reasoning/",
+-        "HF_HOME": "/opt/ml/input/data/training/hf-cache",
++        "HF_HOME": "/opt/ml/data/input/.cache",
+         "HF_TOKEN": get_hf_token(),
+         "WANDB_API_KEY": get_wandb_api_key(),
+         "SM_USE_RESERVED_CAPACITY": "1",

wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/requirements.txt

@@ -0,0 +1,289 @@
+absl-py==2.1.0
+accelerate==1.1.1
+aiohappyeyeballs==2.4.3
+aiohttp==3.10.10
+aiosignal==1.3.1
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+apex==0.1
+appdirs==1.4.4
+argparse==1.4.0
+asttokens==2.4.1
+async-timeout==4.0.3
+attrdict==2.0.1
+attrs==23.2.0
+autocommand==2.2.2
+awscli==1.33.40
+backcall==0.2.0
+backports.tarfile==1.2.0
+bcrypt==4.2.0
+beautifulsoup4==4.12.3
+bleach==6.2.0
+blis==0.7.10
+bokeh==3.5.1
+boltons==23.0.0
+boto3==1.34.158
+botocore==1.34.158
+brotli==1.0.9
+cached-property==1.5.2
+cachetools==5.5.0
+catalogue==2.0.10
+certifi==2024.7.4
+cffi==1.15.1
+chardet==5.2.0
+charset-normalizer==3.2.0
+click==8.1.7
+cloudpathlib==0.18.1
+cloudpickle==2.2.1
+cmake==3.30.2
+colorama==0.4.6
+comm==0.2.2
+conda-libmamba-solver==23.3.0
+conda-package-handling==2.2.0
+conda-package-streaming==0.9.0
+conda==23.3.1
+confection==0.1.4
+contourpy==1.2.1
+cryptography==41.0.7
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.11
+datasets==3.1.0
+debugpy==1.8.5
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.8
+docker-pycreds==0.4.0
+docker==7.1.0
+docopt==0.6.2
+docstring-parser==0.16
+docutils==0.16
+einops==0.8.0
+exceptiongroup==1.2.2
+executing==2.0.1
+fastai==2.7.16
+fastcore==1.5.55
+fastdownload==0.0.7
+fastjsonschema==2.20.0
+fastprogress==1.0.3
+filelock==3.15.4
+fire==0.6.0
+flash-attn==2.5.5
+fonttools==4.53.1
+frozenlist==1.5.0
+fsspec==2024.6.1
+gevent==24.2.1
+gitdb==4.0.11
+gitpython==3.1.43
+gmpy2==2.1.5
+google-pasta==0.2.0
+greenlet==3.0.3
+h5py==3.11.0
+huggingface-hub==0.26.2
+hydra-core==1.3.2
+idna==3.7
+imageio==2.34.2
+importlib-metadata==6.11.0
+importlib-resources==6.1.1
+inflect==7.3.1
+inotify-simple==1.2.1
+ipykernel==6.29.1
+ipython==8.12.3
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+jedi==0.19.1
+jinja2==3.1.4
+jmespath==1.0.1
+joblib==1.4.2
+jsonpatch==1.32
+jsonpointer==2.0
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+jupyter-client==8.6.2
+jupyter-core==5.7.2
+jupyterlab-pygments==0.3.0
+kiwisolver==1.4.5
+langcodes==3.4.0
+language-data==1.2.0
+libmambapy==1.4.2
+llvmlite==0.43.0
+mamba==1.4.2
+marisa-trie==1.1.0
+markdown-it-py==3.0.0
+markupsafe==2.1.5
+matplotlib-inline==0.1.7
+matplotlib==3.9.1
+mdurl==0.1.2
+mistune==3.0.2
+more-itertools==10.3.0
+mpi4py==3.1.6
+mpmath==1.3.0
+multidict==6.1.0
+multiprocess==0.70.16
+munkres==1.1.4
+murmurhash==1.0.10
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==3.3
+ninja==1.11.1.1
+numba==0.60.0
+numpy==1.24.3
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-ml-py==12.535.161
+nvidia-nccl-cu12==2.19.3
+nvidia-nvjitlink-cu12==12.6.77
+nvidia-nvtx-cu12==12.1.105
+nvitop==1.3.2
+omegaconf==2.3.0
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==23.1
+pandas==2.2.2
+pandocfilters==1.5.1
+paramiko==3.4.0
+parso==0.8.4
+pathos==0.3.2
+pexpect==4.9.0
+pickleshare==0.7.5
+pillow==10.3.0
+pip-chill==1.0.3
+pip==24.1.2
+pipreqs==0.5.0
+platformdirs==4.2.2
+plotly==5.23.0
+pluggy==1.3.0
+ply==3.11
+pox==0.3.4
+ppft==1.7.6.8
+preshed==3.0.9
+prompt-toolkit==3.0.47
+propcache==0.2.0
+protobuf==3.20.3
+psutil==6.0.0
+ptyprocess==0.7.0
+pure-eval==0.2.3
+pyarrow==17.0.0
+pyasn1==0.6.0
+pybind11-global==2.13.1
+pybind11==2.13.1
+pycosat==0.6.4
+pycparser==2.21
+pydantic-core==2.20.1
+pydantic==2.8.2
+pyfunctional==1.5.0
+pygments==2.18.0
+pynacl==1.5.0
+pyopenssl==23.2.0
+pyparsing==3.1.2
+pyqt5-sip==12.12.2
+pyqt5==5.15.9
+pysocks==1.7.1
+python-dateutil==2.9.0
+pytz==2024.1
+pyyaml==6.0.2
+pyzmq==26.1.0
+referencing==0.35.1
+regex==2024.11.6
+requests==2.32.3
+retrying==1.3.4
+rich==13.7.1
+rpds-py==0.20.0
+rsa==4.7.2
+ruamel.yaml.clib==0.2.7
+ruamel.yaml==0.17.32
+s3fs==0.4.2
+s3transfer==0.10.2
+safetensors==0.4.5
+sagemaker-experiments==0.1.45
+sagemaker-pytorch-training==2.8.1
+sagemaker-training==4.8.1
+sagemaker==2.224.2
+schema==0.7.7
+scikit-learn==1.3.2
+scipy==1.14.0
+seaborn==0.13.2
+sentencepiece==0.2.0
+sentry-sdk==2.18.0
+setproctitle==1.3.3
+setuptools==72.1.0
+shap==0.46.0
+shellingham==1.5.4
+shtab==1.7.1
+sip==6.7.12
+six==1.16.0
+slicer==0.0.8
+smart-open==7.0.4
+smclarify==0.5
+smdebug-rulesconfig==1.0.1
+smdistributed-dataparallel==2.1.0
+smmap==5.0.1
+smprof==0.3.334
+soupsieve==2.6
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.5
+srsly==2.4.8
+stack-data==0.6.2
+sympy==1.13.0
+tabulate==0.9.0
+tblib==3.0.0
+tenacity==9.0.0
+tensor-parallel==2.0.0
+termcolor==2.5.0
+thinc==8.2.5
+threadpoolctl==3.5.0
+tinycss2==1.4.0
+tokenizers==0.20.3
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.0
+torch==2.2.0
+torchaudio==2.2.0
+torchdata==0.7.0
+torchnet==0.0.4
+torchtext==0.16.0
+torchvision==0.17.0
+tornado==6.4.1
+tqdm==4.66.5
+traitlets==5.14.3
+transformer-engine==0.12.0+170797
+transformers==4.46.0
+triton==2.2.0
+trl==0.12.0
+typeguard==4.3.0
+typer-slim==0.12.3
+typer==0.12.3
+typing-extensions==4.12.2
+tyro==0.8.5
+tzdata==2024.1
+unicodedata2==15.1.0
+urllib3==1.26.19
+visdom==0.2.4
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.4.1
+webencodings==0.5.1
+websocket-client==1.8.0
+werkzeug==3.0.3
+wheel==0.41.2
+wrapt==1.16.0
+xxhash==3.5.0
+xyzservices==2024.6.0
+yarg==0.1.9
+yarl==1.17.1
+zipp==3.19.2
+zope.event==5.0
+zope.interface==7.0.1
+zstandard==0.19.0

wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-metadata.json

@@ -0,0 +1,1069 @@
+{
+    "os": "Linux-5.10.226-214.879.amzn2.x86_64-x86_64-with-glibc2.31",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-11-13T19:02:47.010671",
+    "startedAt": "2024-11-13T19:02:46.426230",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--batch_size",
+        "64",
+        "--cache_dir",
+        "/opt/ml/data/input/.cache",
+        "--dataset",
+        "code-contests",
+        "--gradient_accumulation_steps",
+        "8",
+        "--logging_steps",
+        "5",
+        "--lr",
+        "1e-6",
+        "--max_seq_length",
+        "2048",
+        "--model_name",
+        "Qwen/Qwen2.5-Coder-7B-Instruct",
+        "--num_train_epochs",
+        "2",
+        "--output_dir",
+        "/opt/ml/model/",
+        "--push_to_hub",
+        "True",
+        "--run_name",
+        "sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048",
+        "--save_steps",
+        "100",
+        "--schedule",
+        "cosine",
+        "--wandb_project",
+        "sft-codecontests-1112",
+        "--weight_decay",
+        "0.0"
+    ],
+    "state": "running",
+    "program": "/opt/ml/code/qlearning_reasoning/training/sft.py",
+    "codePathLocal": "qlearning_reasoning/training/sft.py",
+    "codePath": "qlearning_reasoning/training/sft.py",
+    "git": {
+        "remote": "git@github.com:TRI-ML/reasoning-value-verifiers.git",
+        "commit": "cbfd05147f15c55be83f4996eebdb305c17f119d"
+    },
+    "email": null,
+    "root": "/opt/ml/code",
+    "host": "algo-1",
+    "username": "root",
+    "executable": "/opt/conda/bin/python3.10",
+    "cpu_count": 96,
+    "cpu_count_logical": 192,
+    "cpu_freq": {
+        "current": 2829.910838541667,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2850.788,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3406.159,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2605.944,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2608.162,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2818.169,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2591.551,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2621.486,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2607.504,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2999.734,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2991.449,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3123.177,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3053.641,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3003.014,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3000.178,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3432.256,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2999.802,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3005.725,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3565.885,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2992.339,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3104.468,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3182.411,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3013.217,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3131.904,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3007.314,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2625.175,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2653.407,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2610.871,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2793.855,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2619.232,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2627.886,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2613.748,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2318.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3546.661,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3022.56,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3035.28,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3541.443,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3163.55,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3023.79,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3033.204,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3031.695,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3034.587,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3031.71,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3029.116,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3293.701,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3033.78,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3032.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3057.365,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3028.183,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3596.439,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3327.853,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3226.463,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3259.361,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3284.076,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3257.698,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3280.289,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3258.834,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2517.346,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2553.091,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2552.693,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2689.091,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2503.301,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2524.4,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2521.768,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2530.364,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3258.225,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3597.406,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3279.658,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3292.257,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3265.351,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3286.715,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3268.777,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3242.068,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3288.908,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3278.645,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3215.578,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3327.101,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3357.075,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3144.838,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3005.517,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3597.381,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3596.175,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3317.258,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3310.249,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3140.687,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3398.954,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3318.433,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3342.35,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3328.575,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3499.892,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3344.864,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3355.802,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3305.032,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3318.07,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3336.546,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3335.458,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3596.561,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2884.473,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2934.098,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2621.598,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2645.184,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2480.726,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2515.912,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2702.57,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2671.473,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3092.321,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3092.204,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3217.287,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3161.646,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3089.273,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3088.447,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3559.214,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3218.611,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3021.622,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3332.084,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3019.854,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3213.831,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3092.774,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3015.868,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3082.752,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3029.559,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2873.142,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2215.288,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3193.443,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2928.717,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2990.366,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2923.847,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2869.364,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2859.189,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3049.001,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2805.206,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3094.081,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3187.222,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2853.945,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2944.73,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2809.161,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2806.787,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3024.225,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3032.173,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3033.195,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3561.287,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2987.872,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3051.252,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3379.017,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3044.319,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3595.62,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3261.737,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3257.013,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3288.317,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3268.675,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3269.579,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3266.213,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3265.148,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2554.152,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2556.63,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2540.571,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2512.005,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2560.992,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2526.641,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2528.267,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2583.061,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3257.652,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3598.217,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3282.578,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3261.664,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3268.246,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3331.937,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3265.603,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3258.501,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3337.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3259.949,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3226.058,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3250.308,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3208.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2974.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3216.706,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3598.55,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3598.261,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3222.637,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3227.662,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3143.781,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3215.567,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3250.612,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3195.784,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3219.289,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3289.163,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3247.875,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3249.112,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3209.3,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3187.212,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3288.003,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3169.417,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3596.071,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 119.94140625,
+            "used": 56.4015998840332
+        }
+    },
+    "gpu": "NVIDIA H100 80GB HBM3",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 1999.9661331176758
+    }
+}

wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/loss": 0.856, "train/grad_norm": 2.515625, "train/learning_rate": 3.086582838174551e-07, "train/epoch": 1.22, "train/global_step": 15, "_timestamp": 1731525055.2001407, "_runtime": 488.7689197063446, "_step": 6, "eval/loss": 0.7827465534210205, "eval/runtime": 1.6563, "eval/samples_per_second": 61.584, "eval/steps_per_second": 1.208}

wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/logs/debug-internal.log

@@ -0,0 +1,373 @@
+2024-11-13 19:02:46,431 INFO    StreamThr :1939 [internal.py:wandb_internal():86] W&B internal server running at pid: 1939, started at: 2024-11-13 19:02:46.431351
+2024-11-13 19:02:46,433 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status
+2024-11-13 19:02:46,434 INFO    WriterThread:1939 [datastore.py:open_for_write():87] open: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/run-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1.wandb
+2024-11-13 19:02:46,435 DEBUG   SenderThread:1939 [sender.py:send():382] send: header
+2024-11-13 19:02:46,442 DEBUG   SenderThread:1939 [sender.py:send():382] send: run
+2024-11-13 19:02:46,772 INFO    SenderThread:1939 [dir_watcher.py:__init__():211] watching files in: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files
+2024-11-13 19:02:46,772 INFO    SenderThread:1939 [sender.py:_start_run_threads():1136] run started: kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1 with start time 1731524566.431221
+2024-11-13 19:02:46,782 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: check_version
+2024-11-13 19:02:46,782 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: check_version
+2024-11-13 19:02:46,859 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: run_start
+2024-11-13 19:02:46,947 DEBUG   HandlerThread:1939 [system_info.py:__init__():27] System info init
+2024-11-13 19:02:46,947 DEBUG   HandlerThread:1939 [system_info.py:__init__():42] System info init done
+2024-11-13 19:02:46,947 INFO    HandlerThread:1939 [system_monitor.py:start():194] Starting system monitor
+2024-11-13 19:02:46,947 INFO    SystemMonitor:1939 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-11-13 19:02:46,948 INFO    HandlerThread:1939 [system_monitor.py:probe():214] Collecting system info
+2024-11-13 19:02:46,948 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started cpu monitoring
+2024-11-13 19:02:46,949 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started disk monitoring
+2024-11-13 19:02:46,949 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started gpu monitoring
+2024-11-13 19:02:46,951 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started memory monitoring
+2024-11-13 19:02:46,952 INFO    SystemMonitor:1939 [interfaces.py:start():190] Started network monitoring
+2024-11-13 19:02:47,010 DEBUG   HandlerThread:1939 [system_info.py:probe():151] Probing system
+2024-11-13 19:02:47,013 DEBUG   HandlerThread:1939 [system_info.py:_probe_git():136] Probing git
+2024-11-13 19:02:47,021 DEBUG   HandlerThread:1939 [system_info.py:_probe_git():144] Probing git done
+2024-11-13 19:02:47,021 DEBUG   HandlerThread:1939 [system_info.py:probe():199] Probing system done
+2024-11-13 19:02:47,022 DEBUG   HandlerThread:1939 [system_monitor.py:probe():223] {'os': 'Linux-5.10.226-214.879.amzn2.x86_64-x86_64-with-glibc2.31', 'python': '3.10.12', 'heartbeatAt': '2024-11-13T19:02:47.010671', 'startedAt': '2024-11-13T19:02:46.426230', 'docker': None, 'cuda': None, 'args': ('--batch_size', '64', '--cache_dir', '/opt/ml/data/input/.cache', '--dataset', 'code-contests', '--gradient_accumulation_steps', '8', '--logging_steps', '5', '--lr', '1e-6', '--max_seq_length', '2048', '--model_name', 'Qwen/Qwen2.5-Coder-7B-Instruct', '--num_train_epochs', '2', '--output_dir', '/opt/ml/model/', '--push_to_hub', 'True', '--run_name', 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', '--save_steps', '100', '--schedule', 'cosine', '--wandb_project', 'sft-codecontests-1112', '--weight_decay', '0.0'), 'state': 'running', 'program': '/opt/ml/code/qlearning_reasoning/training/sft.py', 'codePathLocal': 'qlearning_reasoning/training/sft.py', 'codePath': 'qlearning_reasoning/training/sft.py', 'git': {'remote': 'git@github.com:TRI-ML/reasoning-value-verifiers.git', 'commit': 'cbfd05147f15c55be83f4996eebdb305c17f119d'}, 'email': None, 'root': '/opt/ml/code', 'host': 'algo-1', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 96, 'cpu_count_logical': 192, 'cpu_freq': {'current': 2829.910838541667, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2850.788, 'min': 0.0, 'max': 0.0}, {'current': 3406.159, 'min': 0.0, 'max': 0.0}, {'current': 2605.944, 'min': 0.0, 'max': 0.0}, {'current': 2608.162, 'min': 0.0, 'max': 0.0}, {'current': 2818.169, 'min': 0.0, 'max': 0.0}, {'current': 2591.551, 'min': 0.0, 'max': 0.0}, {'current': 2621.486, 'min': 0.0, 'max': 0.0}, {'current': 2607.504, 'min': 0.0, 'max': 0.0}, {'current': 2999.734, 'min': 0.0, 'max': 0.0}, {'current': 2991.449, 'min': 0.0, 'max': 0.0}, {'current': 3123.177, 'min': 0.0, 'max': 0.0}, {'current': 3053.641, 'min': 0.0, 'max': 0.0}, {'current': 3003.014, 'min': 0.0, 'max': 0.0}, {'current': 3000.178, 'min': 0.0, 'max': 0.0}, {'current': 3432.256, 'min': 0.0, 'max': 0.0}, {'current': 2999.802, 'min': 0.0, 'max': 0.0}, {'current': 3005.725, 'min': 0.0, 'max': 0.0}, {'current': 3565.885, 'min': 0.0, 'max': 0.0}, {'current': 2992.339, 'min': 0.0, 'max': 0.0}, {'current': 3104.468, 'min': 0.0, 'max': 0.0}, {'current': 3182.411, 'min': 0.0, 'max': 0.0}, {'current': 3013.217, 'min': 0.0, 'max': 0.0}, {'current': 3131.904, 'min': 0.0, 'max': 0.0}, {'current': 3007.314, 'min': 0.0, 'max': 0.0}, {'current': 2625.175, 'min': 0.0, 'max': 0.0}, {'current': 2653.407, 'min': 0.0, 'max': 0.0}, {'current': 2610.871, 'min': 0.0, 'max': 0.0}, {'current': 2793.855, 'min': 0.0, 'max': 0.0}, {'current': 2619.232, 'min': 0.0, 'max': 0.0}, {'current': 2627.886, 'min': 0.0, 'max': 0.0}, {'current': 2613.748, 'min': 0.0, 'max': 0.0}, {'current': 2318.039, 'min': 0.0, 'max': 0.0}, {'current': 3546.661, 'min': 0.0, 'max': 0.0}, {'current': 3022.56, 'min': 0.0, 'max': 0.0}, {'current': 3035.28, 'min': 0.0, 'max': 0.0}, {'current': 3541.443, 'min': 0.0, 'max': 0.0}, {'current': 3163.55, 'min': 0.0, 'max': 0.0}, {'current': 3023.79, 'min': 0.0, 'max': 0.0}, {'current': 3033.204, 'min': 0.0, 'max': 0.0}, {'current': 3031.695, 'min': 0.0, 'max': 0.0}, {'current': 3034.587, 'min': 0.0, 'max': 0.0}, {'current': 3031.71, 'min': 0.0, 'max': 0.0}, {'current': 3029.116, 'min': 0.0, 'max': 0.0}, {'current': 3293.701, 'min': 0.0, 'max': 0.0}, {'current': 3033.78, 'min': 0.0, 'max': 0.0}, {'current': 3032.034, 'min': 0.0, 'max': 0.0}, {'current': 3057.365, 'min': 0.0, 'max': 0.0}, {'current': 3028.183, 'min': 0.0, 'max': 0.0}, {'current': 3596.439, 'min': 0.0, 'max': 0.0}, {'current': 3327.853, 'min': 0.0, 'max': 0.0}, {'current': 3226.463, 'min': 0.0, 'max': 0.0}, {'current': 3259.361, 'min': 0.0, 'max': 0.0}, {'current': 3284.076, 'min': 0.0, 'max': 0.0}, {'current': 3257.698, 'min': 0.0, 'max': 0.0}, {'current': 3280.289, 'min': 0.0, 'max': 0.0}, {'current': 3258.834, 'min': 0.0, 'max': 0.0}, {'current': 2517.346, 'min': 0.0, 'max': 0.0}, {'current': 2553.091, 'min': 0.0, 'max': 0.0}, {'current': 2552.693, 'min': 0.0, 'max': 0.0}, {'current': 2689.091, 'min': 0.0, 'max': 0.0}, {'current': 2503.301, 'min': 0.0, 'max': 0.0}, {'current': 2524.4, 'min': 0.0, 'max': 0.0}, {'current': 2521.768, 'min': 0.0, 'max': 0.0}, {'current': 2530.364, 'min': 0.0, 'max': 0.0}, {'current': 3258.225, 'min': 0.0, 'max': 0.0}, {'current': 3597.406, 'min': 0.0, 'max': 0.0}, {'current': 3279.658, 'min': 0.0, 'max': 0.0}, {'current': 3292.257, 'min': 0.0, 'max': 0.0}, {'current': 3265.351, 'min': 0.0, 'max': 0.0}, {'current': 3286.715, 'min': 0.0, 'max': 0.0}, {'current': 3268.777, 'min': 0.0, 'max': 0.0}, {'current': 3242.068, 'min': 0.0, 'max': 0.0}, {'current': 3288.908, 'min': 0.0, 'max': 0.0}, {'current': 3278.645, 'min': 0.0, 'max': 0.0}, {'current': 3215.578, 'min': 0.0, 'max': 0.0}, {'current': 3327.101, 'min': 0.0, 'max': 0.0}, {'current': 3357.075, 'min': 0.0, 'max': 0.0}, {'current': 3144.838, 'min': 0.0, 'max': 0.0}, {'current': 3005.517, 'min': 0.0, 'max': 0.0}, {'current': 3597.381, 'min': 0.0, 'max': 0.0}, {'current': 3596.175, 'min': 0.0, 'max': 0.0}, {'current': 3317.258, 'min': 0.0, 'max': 0.0}, {'current': 3310.249, 'min': 0.0, 'max': 0.0}, {'current': 3140.687, 'min': 0.0, 'max': 0.0}, {'current': 3398.954, 'min': 0.0, 'max': 0.0}, {'current': 3318.433, 'min': 0.0, 'max': 0.0}, {'current': 3342.35, 'min': 0.0, 'max': 0.0}, {'current': 3328.575, 'min': 0.0, 'max': 0.0}, {'current': 3499.892, 'min': 0.0, 'max': 0.0}, {'current': 3344.864, 'min': 0.0, 'max': 0.0}, {'current': 3355.802, 'min': 0.0, 'max': 0.0}, {'current': 3305.032, 'min': 0.0, 'max': 0.0}, {'current': 3318.07, 'min': 0.0, 'max': 0.0}, {'current': 3336.546, 'min': 0.0, 'max': 0.0}, {'current': 3335.458, 'min': 0.0, 'max': 0.0}, {'current': 3596.561, 'min': 0.0, 'max': 0.0}, {'current': 2884.473, 'min': 0.0, 'max': 0.0}, {'current': 2934.098, 'min': 0.0, 'max': 0.0}, {'current': 2621.598, 'min': 0.0, 'max': 0.0}, {'current': 2645.184, 'min': 0.0, 'max': 0.0}, {'current': 2480.726, 'min': 0.0, 'max': 0.0}, {'current': 2515.912, 'min': 0.0, 'max': 0.0}, {'current': 2702.57, 'min': 0.0, 'max': 0.0}, {'current': 2671.473, 'min': 0.0, 'max': 0.0}, {'current': 3092.321, 'min': 0.0, 'max': 0.0}, {'current': 3092.204, 'min': 0.0, 'max': 0.0}, {'current': 3217.287, 'min': 0.0, 'max': 0.0}, {'current': 3161.646, 'min': 0.0, 'max': 0.0}, {'current': 3089.273, 'min': 0.0, 'max': 0.0}, {'current': 3088.447, 'min': 0.0, 'max': 0.0}, {'current': 3559.214, 'min': 0.0, 'max': 0.0}, {'current': 3218.611, 'min': 0.0, 'max': 0.0}, {'current': 3021.622, 'min': 0.0, 'max': 0.0}, {'current': 3332.084, 'min': 0.0, 'max': 0.0}, {'current': 3019.854, 'min': 0.0, 'max': 0.0}, {'current': 3213.831, 'min': 0.0, 'max': 0.0}, {'current': 3092.774, 'min': 0.0, 'max': 0.0}, {'current': 3015.868, 'min': 0.0, 'max': 0.0}, {'current': 3082.752, 'min': 0.0, 'max': 0.0}, {'current': 3029.559, 'min': 0.0, 'max': 0.0}, {'current': 2873.142, 'min': 0.0, 'max': 0.0}, {'current': 2215.288, 'min': 0.0, 'max': 0.0}, {'current': 3193.443, 'min': 0.0, 'max': 0.0}, {'current': 2928.717, 'min': 0.0, 'max': 0.0}, {'current': 2990.366, 'min': 0.0, 'max': 0.0}, {'current': 2923.847, 'min': 0.0, 'max': 0.0}, {'current': 2869.364, 'min': 0.0, 'max': 0.0}, {'current': 2859.189, 'min': 0.0, 'max': 0.0}, {'current': 3049.001, 'min': 0.0, 'max': 0.0}, {'current': 2805.206, 'min': 0.0, 'max': 0.0}, {'current': 3094.081, 'min': 0.0, 'max': 0.0}, {'current': 3187.222, 'min': 0.0, 'max': 0.0}, {'current': 2853.945, 'min': 0.0, 'max': 0.0}, {'current': 2944.73, 'min': 0.0, 'max': 0.0}, {'current': 2809.161, 'min': 0.0, 'max': 0.0}, {'current': 2806.787, 'min': 0.0, 'max': 0.0}, {'current': 3024.225, 'min': 0.0, 'max': 0.0}, {'current': 3032.173, 'min': 0.0, 'max': 0.0}, {'current': 3033.195, 'min': 0.0, 'max': 0.0}, {'current': 3561.287, 'min': 0.0, 'max': 0.0}, {'current': 2987.872, 'min': 0.0, 'max': 0.0}, {'current': 3051.252, 'min': 0.0, 'max': 0.0}, {'current': 3379.017, 'min': 0.0, 'max': 0.0}, {'current': 3044.319, 'min': 0.0, 'max': 0.0}, {'current': 3595.62, 'min': 0.0, 'max': 0.0}, {'current': 3261.737, 'min': 0.0, 'max': 0.0}, {'current': 3257.013, 'min': 0.0, 'max': 0.0}, {'current': 3288.317, 'min': 0.0, 'max': 0.0}, {'current': 3268.675, 'min': 0.0, 'max': 0.0}, {'current': 3269.579, 'min': 0.0, 'max': 0.0}, {'current': 3266.213, 'min': 0.0, 'max': 0.0}, {'current': 3265.148, 'min': 0.0, 'max': 0.0}, {'current': 2554.152, 'min': 0.0, 'max': 0.0}, {'current': 2556.63, 'min': 0.0, 'max': 0.0}, {'current': 2540.571, 'min': 0.0, 'max': 0.0}, {'current': 2512.005, 'min': 0.0, 'max': 0.0}, {'current': 2560.992, 'min': 0.0, 'max': 0.0}, {'current': 2526.641, 'min': 0.0, 'max': 0.0}, {'current': 2528.267, 'min': 0.0, 'max': 0.0}, {'current': 2583.061, 'min': 0.0, 'max': 0.0}, {'current': 3257.652, 'min': 0.0, 'max': 0.0}, {'current': 3598.217, 'min': 0.0, 'max': 0.0}, {'current': 3282.578, 'min': 0.0, 'max': 0.0}, {'current': 3261.664, 'min': 0.0, 'max': 0.0}, {'current': 3268.246, 'min': 0.0, 'max': 0.0}, {'current': 3331.937, 'min': 0.0, 'max': 0.0}, {'current': 3265.603, 'min': 0.0, 'max': 0.0}, {'current': 3258.501, 'min': 0.0, 'max': 0.0}, {'current': 3337.045, 'min': 0.0, 'max': 0.0}, {'current': 3259.949, 'min': 0.0, 'max': 0.0}, {'current': 3226.058, 'min': 0.0, 'max': 0.0}, {'current': 3250.308, 'min': 0.0, 'max': 0.0}, {'current': 3208.026, 'min': 0.0, 'max': 0.0}, {'current': 2974.043, 'min': 0.0, 'max': 0.0}, {'current': 3216.706, 'min': 0.0, 'max': 0.0}, {'current': 3598.55, 'min': 0.0, 'max': 0.0}, {'current': 3598.261, 'min': 0.0, 'max': 0.0}, {'current': 3222.637, 'min': 0.0, 'max': 0.0}, {'current': 3227.662, 'min': 0.0, 'max': 0.0}, {'current': 3143.781, 'min': 0.0, 'max': 0.0}, {'current': 3215.567, 'min': 0.0, 'max': 0.0}, {'current': 3250.612, 'min': 0.0, 'max': 0.0}, {'current': 3195.784, 'min': 0.0, 'max': 0.0}, {'current': 3219.289, 'min': 0.0, 'max': 0.0}, {'current': 3289.163, 'min': 0.0, 'max': 0.0}, {'current': 3247.875, 'min': 0.0, 'max': 0.0}, {'current': 3249.112, 'min': 0.0, 'max': 0.0}, {'current': 3209.3, 'min': 0.0, 'max': 0.0}, {'current': 3187.212, 'min': 0.0, 'max': 0.0}, {'current': 3288.003, 'min': 0.0, 'max': 0.0}, {'current': 3169.417, 'min': 0.0, 'max': 0.0}, {'current': 3596.071, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 119.94140625, 'used': 56.4015998840332}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9661331176758}}
+2024-11-13 19:02:47,022 INFO    HandlerThread:1939 [system_monitor.py:probe():224] Finished collecting system info
+2024-11-13 19:02:47,022 INFO    HandlerThread:1939 [system_monitor.py:probe():227] Publishing system info
+2024-11-13 19:02:47,022 DEBUG   HandlerThread:1939 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment
+2024-11-13 19:02:47,775 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/conda-environment.yaml
+2024-11-13 19:03:01,345 DEBUG   HandlerThread:1939 [system_info.py:_save_conda():220] Saving conda packages done
+2024-11-13 19:03:01,345 DEBUG   HandlerThread:1939 [system_info.py:_save_code():45] Saving code
+2024-11-13 19:03:01,349 DEBUG   HandlerThread:1939 [system_info.py:_save_code():66] Saving code done
+2024-11-13 19:03:01,349 DEBUG   HandlerThread:1939 [system_info.py:_save_patches():83] Saving git patches
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/conda-environment.yaml
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/code/qlearning_reasoning/training/sft.py
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/code/qlearning_reasoning
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/code/qlearning_reasoning/training
+2024-11-13 19:03:01,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/code
+2024-11-13 19:03:01,989 DEBUG   HandlerThread:1939 [system_info.py:_save_patches():125] Saving git patches done
+2024-11-13 19:03:01,991 INFO    HandlerThread:1939 [system_monitor.py:probe():229] Finished publishing system info
+2024-11-13 19:03:01,993 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:01,993 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: keepalive
+2024-11-13 19:03:01,994 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:01,994 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: keepalive
+2024-11-13 19:03:01,994 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:01,994 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: keepalive
+2024-11-13 19:03:01,994 DEBUG   SenderThread:1939 [sender.py:send():382] send: files
+2024-11-13 19:03:01,995 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-11-13 19:03:01,995 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file code/qlearning_reasoning/training/sft.py with policy now
+2024-11-13 19:03:01,995 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file diff.patch with policy now
+2024-11-13 19:03:01,999 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: python_packages
+2024-11-13 19:03:01,999 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: python_packages
+2024-11-13 19:03:02,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:03:02,003 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:03:02,003 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:03:02,111 DEBUG   SenderThread:1939 [sender.py:send():382] send: telemetry
+2024-11-13 19:03:02,111 DEBUG   SenderThread:1939 [sender.py:send():382] send: config
+2024-11-13 19:03:02,112 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:02,112 DEBUG   SenderThread:1939 [sender.py:send():382] send: telemetry
+2024-11-13 19:03:02,112 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:02,113 WARNING SenderThread:1939 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen)
+2024-11-13 19:03:02,113 DEBUG   SenderThread:1939 [sender.py:send():382] send: telemetry
+2024-11-13 19:03:02,113 DEBUG   SenderThread:1939 [sender.py:send():382] send: telemetry
+2024-11-13 19:03:02,113 DEBUG   SenderThread:1939 [sender.py:send():382] send: config
+2024-11-13 19:03:02,551 INFO    wandb-upload_1:1939 [upload_job.py:push():131] Uploaded file /tmp/tmpm_h7h9v0wandb/pjhigh2e-code/qlearning_reasoning/training/sft.py
+2024-11-13 19:03:02,554 INFO    wandb-upload_0:1939 [upload_job.py:push():131] Uploaded file /tmp/tmpm_h7h9v0wandb/f98paeu8-wandb-metadata.json
+2024-11-13 19:03:02,556 INFO    wandb-upload_2:1939 [upload_job.py:push():131] Uploaded file /tmp/tmpm_h7h9v0wandb/cc8zuviy-diff.patch
+2024-11-13 19:03:02,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/diff.patch
+2024-11-13 19:03:02,776 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:03:02,777 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/requirements.txt
+2024-11-13 19:03:02,777 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-metadata.json
+2024-11-13 19:03:04,777 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:03:07,114 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:12,115 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:16,999 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:03:17,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:03:17,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:03:18,090 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:18,782 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/config.yaml
+2024-11-13 19:03:23,250 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:26,095 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:03:26,097 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:03:26,098 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:03:26,099 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:03:26,785 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_created():271] file/dir created: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:03:28,785 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:03:29,100 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:31,999 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:03:32,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:03:32,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:03:34,147 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:39,147 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:44,148 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:46,955 DEBUG   SystemMonitor:1939 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-11-13 19:03:46,959 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:03:47,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:03:47,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:03:47,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:03:49,636 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:50,813 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:03:50,814 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/config.yaml
+2024-11-13 19:03:54,814 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:03:59,814 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:02,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:04:02,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:04:02,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:04:05,061 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:10,062 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:14,822 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:04:15,419 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:16,962 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:04:17,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:04:17,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:04:17,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:04:21,087 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:26,088 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:31,089 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:32,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:04:32,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:04:32,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:04:36,587 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:36,829 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:04:41,588 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:46,588 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:46,965 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:04:47,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:04:47,000 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:04:47,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:04:52,111 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:57,111 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:04:57,383 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:04:57,384 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:04:57,385 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:04:57,385 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:04:57,837 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:04:58,837 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:04:59,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:04:59,004 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:04:59,005 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:04:59,005 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:04:59,005 DEBUG   SenderThread:1939 [sender.py:send():382] send: metric
+2024-11-13 19:04:59,006 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:04:59,006 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:04:59,006 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:04:59,838 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:05:00,838 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:05:02,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:05:02,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:05:02,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:05:03,111 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:08,112 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:13,113 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:16,967 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:05:17,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:05:17,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:05:17,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:05:18,123 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:22,846 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:05:23,982 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:25,847 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/config.yaml
+2024-11-13 19:05:30,486 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:32,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:05:32,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:05:32,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:05:36,117 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:41,118 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:46,513 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:46,854 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:05:46,970 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:05:47,000 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:05:47,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:05:47,003 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:05:52,097 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:05:57,098 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:02,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:06:02,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:06:02,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:06:02,152 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:07,153 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:08,862 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:06:12,639 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:16,973 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:06:17,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:06:17,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:06:17,004 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:06:18,127 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:23,127 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:28,128 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:30,870 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:06:32,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:06:32,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:06:32,003 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:06:33,143 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:38,144 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:43,144 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:46,975 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:06:47,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:06:47,001 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:06:47,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:06:49,137 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:51,931 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:06:51,932 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:06:51,933 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:06:51,934 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:06:52,878 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:06:52,878 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:06:53,541 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:06:53,543 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:06:53,543 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:06:53,544 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:06:53,878 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:06:54,545 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:06:54,878 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:06:59,546 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:02,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:07:02,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:07:02,004 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:07:05,164 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:10,165 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:15,165 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:16,886 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:07:16,978 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:07:17,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:07:17,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:07:17,003 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:07:21,108 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:26,108 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:31,109 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:32,001 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:07:32,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:07:32,043 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:07:36,934 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:38,894 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:07:41,935 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:46,935 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:46,981 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:07:47,002 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:07:47,002 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:07:47,004 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:07:52,135 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:07:57,136 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:02,136 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:02,856 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:08:02,923 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:08:02,924 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:08:08,076 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:13,077 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:16,985 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:08:17,407 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:08:17,407 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:08:17,447 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:08:18,545 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:23,546 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:28,546 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:32,914 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:33,843 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:34,729 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:08:34,729 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:08:34,771 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:08:39,802 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:44,803 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:46,987 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:08:48,007 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:08:48,007 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:08:48,008 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:08:48,921 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:49,945 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:50,922 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:52,923 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:54,924 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:55,154 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:08:56,925 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:08:58,926 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:09:00,371 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:00,948 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:09:03,950 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:09:05,541 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:05,782 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:09:06,102 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:09:06,102 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:09:11,263 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:16,264 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:16,990 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:09:20,961 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:09:22,073 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:22,272 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:09:22,272 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:09:27,401 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:32,401 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:36,126 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:09:36,666 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:09:36,666 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:09:38,052 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:39,964 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:09:43,769 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:46,992 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:09:47,846 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:09:47,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:09:47,847 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:09:49,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:54,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:09:59,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:02,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:10:02,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:10:02,848 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:10:05,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:10,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:11,977 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:15,248 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:16,996 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:10:17,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:10:17,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:10:17,848 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:10:21,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:26,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:31,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:32,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:10:32,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:10:32,847 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:10:33,985 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:37,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:42,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:46,998 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:10:47,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:10:47,847 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:10:47,887 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:10:47,921 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:53,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:10:53,575 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:10:53,576 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:10:53,576 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:10:53,577 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:10:53,992 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:10:54,993 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:55,200 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: partial_history
+2024-11-13 19:10:55,201 DEBUG   SenderThread:1939 [sender.py:send():382] send: history
+2024-11-13 19:10:55,201 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: summary_record
+2024-11-13 19:10:55,203 INFO    SenderThread:1939 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-11-13 19:10:55,993 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:55,994 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/wandb-summary.json
+2024-11-13 19:10:56,994 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:57,994 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:10:59,204 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:02,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:11:02,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:11:02,849 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:11:04,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:09,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:14,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:17,000 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:11:17,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:11:17,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:11:17,848 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:11:19,311 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:20,004 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:11:24,312 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:29,312 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:32,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:11:32,848 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:11:32,849 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:11:35,231 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:40,232 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:44,014 INFO    Thread-12 :1939 [dir_watcher.py:_on_file_modified():288] file/dir modified: /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/files/output.log
+2024-11-13 19:11:46,173 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report
+2024-11-13 19:11:47,003 DEBUG   SenderThread:1939 [sender.py:send():382] send: stats
+2024-11-13 19:11:47,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: internal_messages
+2024-11-13 19:11:47,847 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: stop_status
+2024-11-13 19:11:47,848 DEBUG   SenderThread:1939 [sender.py:send_request():409] send_request: stop_status
+2024-11-13 19:11:51,230 DEBUG   HandlerThread:1939 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/logs/debug.log

@@ -0,0 +1,30 @@
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Configure stats pid to 173
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Loading settings from /opt/ml/code/wandb/settings
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'api_key': '***REDACTED***', 'project': 'sft-codecontests-1112', 'run_name': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048'}
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'qlearning_reasoning/training/sft.py', 'program_abspath': '/opt/ml/code/qlearning_reasoning/training/sft.py', 'program': '/opt/ml/code/qlearning_reasoning/training/sft.py'}
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:_log_setup():526] Logging user logs to /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/logs/debug.log
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:_log_setup():527] Logging internal logs to /opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/wandb/run-20241113_190246-kushalarora-rvv-main-2024-11-13-17-55-42-664-g0ro8r-algo-1/logs/debug-internal.log
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:init():566] calling init triggers
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sagemaker_training_job_name': 'kushalarora-rvv-main-2024-11-13-17-55-42-664', 'batch_size': 64, 'cache_dir': '/opt/ml/data/input/.cache', 'dataset': 'code-contests', 'gradient_accumulation_steps': 8, 'logging_steps': 5, 'lr': '1e-6', 'max_seq_length': 2048, 'model_name': 'Qwen/Qwen2.5-Coder-7B-Instruct', 'num_train_epochs': 2, 'output_dir': '/opt/ml/output', 'push_to_hub': 'true', 'run_name': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'sagemaker_container_log_level': 20, 'sagemaker_instance_type': 'ml.p5.48xlarge', 'sagemaker_job_name': 'kushalarora-rvv-main-2024-11-13-17-55-42-664', 'sagemaker_program': 'sft.py', 'sagemaker_region': 'us-east-1', 'sagemaker_submit_directory': 's3://tri-ml-datasets/kushalarora-rvv-main-2024-11-13-17-55-42-664/source/sourcedir.tar.gz', 'sagemaker_torch_distributed_enabled': 'true', 'save_steps': 100, 'schedule': 'cosine', 'wandb_project': 'sft-codecontests-1112', 'weight_decay': 0.0, 'additional_framework_parameters': {'sagemaker_instance_type': 'ml.p5.48xlarge', 'sagemaker_torch_distributed_enabled': True}, 'channel_input_dirs': {}, 'current_host': 'algo-1', 'current_instance_group': 'homogeneousCluster', 'current_instance_group_hosts': ['algo-1'], 'current_instance_type': 'ml.p5.48xlarge', 'distribution_hosts': ['algo-1'], 'distribution_instance_groups': ['homogeneousCluster'], 'framework_module': 'sagemaker_pytorch_container.training:main', 'hosts': ['algo-1'], 'hyperparameters': {'batch_size': 64, 'cache_dir': '/opt/ml/data/input/.cache', 'dataset': 'code-contests', 'gradient_accumulation_steps': 8, 'logging_steps': 5, 'lr': '1e-6', 'max_seq_length': 2048, 'model_name': 'Qwen/Qwen2.5-Coder-7B-Instruct', 'num_train_epochs': 2, 'output_dir': '/opt/ml/model/', 'push_to_hub': True, 'run_name': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'save_steps': 100, 'schedule': 'cosine', 'wandb_project': 'sft-codecontests-1112', 'weight_decay': 0.0}, 'input_config_dir': '/opt/ml/input/config', 'input_data_config': {}, 'input_dir': '/opt/ml/input', 'instance_groups': ['homogeneousCluster'], 'instance_groups_dict': {'homogeneousCluster': {'hosts': ['algo-1'], 'instance_group_name': 'homogeneousCluster', 'instance_type': 'ml.p5.48xlarge'}}, 'is_hetero': False, 'is_master': True, 'is_modelparallel_enabled': None, 'is_smddpmprun_installed': False, 'is_smddprun_installed': True, 'job_name': 'kushalarora-rvv-main-2024-11-13-17-55-42-664', 'log_level': 20, 'master_hostname': 'algo-1', 'model_dir': '/opt/ml/model', 'module_dir': 's3://tri-ml-datasets/kushalarora-rvv-main-2024-11-13-17-55-42-664/source/sourcedir.tar.gz', 'module_name': '/opt/ml/code/qlearning_reasoning/training/sft', 'network_interface_name': 'eth0', 'num_cpus': 192, 'num_gpus': 8, 'num_neurons': 0, 'output_data_dir': '/opt/ml/output/data', 'output_intermediate_dir': '/opt/ml/output/intermediate', 'resource_config': {'current_group_name': 'homogeneousCluster', 'current_host': 'algo-1', 'current_instance_type': 'ml.p5.48xlarge', 'hosts': ['algo-1'], 'instance_groups': [{'hosts': ['algo-1'], 'instance_group_name': 'homogeneousCluster', 'instance_type': 'ml.p5.48xlarge'}], 'network_interface_name': 'eth0'}, 'user_entry_point': '/opt/ml/code/qlearning_reasoning/training/sft.py'}
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:init():616] starting backend
+2024-11-13 19:02:46,428 INFO    MainThread:173 [wandb_init.py:init():620] setting up manager
+2024-11-13 19:02:46,429 INFO    MainThread:173 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-13 19:02:46,431 INFO    MainThread:173 [wandb_init.py:init():628] backend started and connected
+2024-11-13 19:02:46,434 INFO    MainThread:173 [wandb_init.py:init():720] updated telemetry
+2024-11-13 19:02:46,440 INFO    MainThread:173 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-11-13 19:02:46,781 INFO    MainThread:173 [wandb_run.py:_on_init():2262] communicating current version
+2024-11-13 19:02:46,812 INFO    MainThread:173 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.18.6 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-11-13 19:02:46,813 INFO    MainThread:173 [wandb_init.py:init():804] starting run threads in backend
+2024-11-13 19:03:02,000 INFO    MainThread:173 [wandb_run.py:_console_start():2241] atexit reg
+2024-11-13 19:03:02,000 INFO    MainThread:173 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-11-13 19:03:02,000 INFO    MainThread:173 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-11-13 19:03:02,001 INFO    MainThread:173 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-11-13 19:03:02,001 INFO    MainThread:173 [wandb_init.py:init():847] run started, returning control to user process
+2024-11-13 19:03:02,004 INFO    MainThread:173 [wandb_run.py:_config_callback():1343] config_cb None None {'vocab_size': 151671, 'max_position_embeddings': 32768, 'hidden_size': 3584, 'intermediate_size': 18944, 'num_hidden_layers': 28, 'num_attention_heads': 28, 'use_sliding_window': False, 'sliding_window': None, 'max_window_layers': 28, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 1000000.0, 'rope_scaling': None, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['Qwen2ForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': 151643, 'eos_token_id': 151645, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'Qwen/Qwen2.5-Coder-7B-Instruct', '_attn_implementation_autoset': True, 'transformers_version': '4.46.0', 'model_type': 'qwen2', 'output_dir': '/opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 8, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-06, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/opt/ml/model//sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048/runs/Nov13_19-02-15_algo-1', 'logging_strategy': 'steps', 'logging_first_step': True, 'logging_steps': 5, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 100, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_acc-8_len-2048', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': ['full_shard', 'auto_wrap'], 'fsdp_min_num_params': 0, 'fsdp_config': {'activation_checkpointing': False, 'auto_wrap_policy': 'TRANSFORMER_BASED_WRAP', 'backward_prefetch': 'BACKWARD_PRE', 'cpu_ram_efficient_loading': True, 'forward_prefetch': False, 'offload_params': False, 'sharding_strategy': 'FULL_SHARD', 'state_dict_type': 'FULL_STATE_DICT', 'sync_module_states': True, 'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'sft-codecontests-qwen_ds-code-contests_model-Qwen2.5-Coder-7B-Instruct_sch-cosine_lr-1e-6_bs-64_', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': {'use_reentrant': False}, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False, 'dataset_text_field': 'text', 'packing': False, 'max_seq_length': 2048, 'dataset_num_proc': None, 'dataset_batch_size': 64, 'model_init_kwargs': None, 'dataset_kwargs': {}, 'eval_packing': None, 'num_of_sequences': 1024, 'chars_per_token': '<CHARS_PER_TOKEN>', 'use_liger': False}
+2024-11-13 19:03:02,008 INFO    MainThread:173 [wandb_config.py:__setitem__():151] config set model/num_parameters = 951599936 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7fb464957310>>
+2024-11-13 19:03:02,009 INFO    MainThread:173 [wandb_run.py:_config_callback():1343] config_cb model/num_parameters 951599936 None