Upload folder using huggingface_hub

args.yaml

@@ -0,0 +1,42 @@
+batch_size: 1
+checkpoint: true
+ckpt_freq: 10
+data:
+  data: ''
+  eval_instruct_data: ./data/t_eval.jsonl
+  instruct:
+    dynamic_chunk_fn_call: true
+    shuffle: true
+  instruct_data: ./data/t_train.jsonl
+  shuffle: false
+eval_freq: 5
+log_freq: 1
+lora:
+  dropout: 0.05
+  enable: true
+  rank: 16
+  scaling: 2.0
+max_norm: 1.0
+max_steps: 10
+mlflow:
+  experiment_name: null
+  tracking_uri: null
+model_id_or_path: ../downloaded_model/mistral_models/7B-v0.3/
+no_ckpt: false
+no_eval: false
+num_ckpt_keep: 3
+num_microbatches: 4
+optim:
+  lr: 0.0001
+  pct_start: 0.05
+  weight_decay: 0.1
+run_dir: ./content/test_ultra_1
+save_adapters: false
+seed: 0
+seq_len: 1024
+wandb:
+  key: 9e930bac040e715c4be9de064318956067b45479
+  offline: false
+  project: finetuning_mistral
+  run_name: null
+world_size: 1

checkpoints/checkpoint_000010/consolidated/consolidated.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1168a65dcc1f06d0371b8a2834c8936502becc2dab70ef51f56f6ac9e8eaa05
+size 14496078512

checkpoints/checkpoint_000010/consolidated/params.json

@@ -0,0 +1,18 @@
+{
+    "dim": 4096,
+    "n_layers": 32,
+    "head_dim": 128,
+    "hidden_dim": 14336,
+    "n_heads": 32,
+    "n_kv_heads": 8,
+    "norm_eps": 1e-05,
+    "vocab_size": 32768,
+    "rope_theta": 1000000.0,
+    "lora": {
+        "enable": true,
+        "rank": 16,
+        "dropout": 0.05,
+        "scaling": 2.0
+    },
+    "moe": null
+}

metrics.eval.jsonl

@@ -0,0 +1,2 @@
+{"step": 5, "train_loss": 0.30592161417007446, "perplexity": 1.260413646697998, "eval_loss": 0.33389732241630554, "at": "2024-09-01T08:08:52.332862"}
+{"step": 10, "train_loss": 0.108039490878582, "perplexity": 1.1342318058013916, "eval_loss": 0.18171557784080505, "at": "2024-09-01T08:09:07.247441"}

metrics.train.jsonl

@@ -0,0 +1,10 @@
+{"lr": 9.931806789787545e-05, "step": 1, "loss": 1.1353108882904053, "percent_done": 10.0, "peak_allocated_mem": 14.656035900115967, "allocated_mem": 14.327418327331543, "wps": 1177.8248031910382, "avg_wps": 1177.8248031910382, "eta_in_seconds": 31.298372983932495, "at": "2024-09-01T08:08:40.093343"}
+{"lr": 9.397371166557421e-05, "step": 2, "loss": 0.882387101650238, "percent_done": 20.0, "peak_allocated_mem": 14.968535900115967, "allocated_mem": 14.327418327331543, "wps": 1420.5461474296683, "avg_wps": 1287.848823987544, "eta_in_seconds": 25.44398021697998, "at": "2024-09-01T08:08:42.978158"}
+{"lr": 8.386414312497274e-05, "step": 3, "loss": 0.7568908929824829, "percent_done": 30.0, "peak_allocated_mem": 14.968535900115967, "allocated_mem": 14.327418327331543, "wps": 1376.892615343285, "avg_wps": 1316.2222344521215, "eta_in_seconds": 21.78355542818705, "at": "2024-09-01T08:08:45.954286"}
+{"lr": 7.008489089356355e-05, "step": 4, "loss": 0.4648980498313904, "percent_done": 40.0, "peak_allocated_mem": 14.968535900115967, "allocated_mem": 14.327418327331543, "wps": 1424.7641809898853, "avg_wps": 1341.7771840572461, "eta_in_seconds": 18.316006779670715, "at": "2024-09-01T08:08:48.830255"}
+{"lr": 5.412915075774753e-05, "step": 5, "loss": 0.30592161417007446, "percent_done": 50.0, "peak_allocated_mem": 14.968535900115967, "allocated_mem": 14.327418327331543, "wps": 1169.4011792663457, "avg_wps": 1303.3529414843938, "eta_in_seconds": 15.713318586349487, "at": "2024-09-01T08:08:52.333812"}
+{"lr": 3.772597474005748e-05, "step": 6, "loss": 0.22509340941905975, "percent_done": 60.0, "peak_allocated_mem": 14.968535900115967, "allocated_mem": 14.327418327331543, "wps": 1470.749009377081, "avg_wps": 1328.554938478532, "eta_in_seconds": 12.33219607671102, "at": "2024-09-01T08:08:55.120107"}
+{"lr": 2.265290148351029e-05, "step": 7, "loss": 0.16402409970760345, "percent_done": 70.0, "peak_allocated_mem": 14.968535900115967, "allocated_mem": 14.327418327331543, "wps": 1418.1967238745826, "avg_wps": 1340.6607771976503, "eta_in_seconds": 9.165629523141043, "at": "2024-09-01T08:08:58.009620"}
+{"lr": 1.0543332358282206e-05, "step": 8, "loss": 0.12998557090759277, "percent_done": 80.0, "peak_allocated_mem": 14.968535900115967, "allocated_mem": 14.327418327331543, "wps": 1411.0965588366037, "avg_wps": 1349.0782897373429, "eta_in_seconds": 6.072293996810913, "at": "2024-09-01T08:09:00.913464"}
+{"lr": 2.7095270784166083e-06, "step": 9, "loss": 0.15246960520744324, "percent_done": 90.0, "peak_allocated_mem": 14.968535900115967, "allocated_mem": 14.327418327331543, "wps": 1411.9670613026851, "avg_wps": 1355.7879077002867, "eta_in_seconds": 3.0211215019226074, "at": "2024-09-01T08:09:03.815685"}
+{"lr": 3.9999999999999996e-10, "step": 10, "loss": 0.108039490878582, "percent_done": 100.0, "peak_allocated_mem": 14.968535900115967, "allocated_mem": 14.327418327331543, "wps": 1193.533250331889, "avg_wps": 1337.6038764958669, "eta_in_seconds": 0.0, "at": "2024-09-01T08:09:07.248545"}

tb/events.out.tfevents.1725178109.f6e183d76cb1.1234.0.train

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46ad0922957fe7b47b48fe24c04a06c063ea0de82bc87a409ef40df49995711d
+size 4338

tb/events.out.tfevents.1725178110.f6e183d76cb1.1234.1.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:692f396a6274e05cbc16f1af6e15d41684082b2c5192ddae6c185fef0bcd3c2f
+size 404

wandb/debug-internal.log

@@ -0,0 +1,266 @@
+2024-09-01 08:08:30,433 INFO    StreamThr :1314 [internal.py:wandb_internal():85] W&B internal server running at pid: 1314, started at: 2024-09-01 08:08:30.432847
+2024-09-01 08:08:30,434 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status
+2024-09-01 08:08:30,436 INFO    WriterThread:1314 [datastore.py:open_for_write():87] open: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/run-ra4ttuu9.wandb
+2024-09-01 08:08:30,437 DEBUG   SenderThread:1314 [sender.py:send():391] send: header
+2024-09-01 08:08:30,442 DEBUG   SenderThread:1314 [sender.py:send():391] send: run
+2024-09-01 08:08:30,661 INFO    SenderThread:1314 [dir_watcher.py:__init__():211] watching files in: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files
+2024-09-01 08:08:30,661 INFO    SenderThread:1314 [sender.py:_start_run_threads():1200] run started: ra4ttuu9 with start time 1725178110.432525
+2024-09-01 08:08:30,668 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: run_start
+2024-09-01 08:08:30,682 DEBUG   HandlerThread:1314 [system_info.py:__init__():26] System info init
+2024-09-01 08:08:30,682 DEBUG   HandlerThread:1314 [system_info.py:__init__():41] System info init done
+2024-09-01 08:08:30,682 INFO    HandlerThread:1314 [system_monitor.py:start():194] Starting system monitor
+2024-09-01 08:08:30,683 INFO    SystemMonitor:1314 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-09-01 08:08:30,683 INFO    HandlerThread:1314 [system_monitor.py:probe():214] Collecting system info
+2024-09-01 08:08:30,684 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started cpu monitoring
+2024-09-01 08:08:30,684 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started disk monitoring
+2024-09-01 08:08:30,685 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started gpu monitoring
+2024-09-01 08:08:30,686 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started memory monitoring
+2024-09-01 08:08:30,686 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started network monitoring
+2024-09-01 08:08:30,709 DEBUG   HandlerThread:1314 [system_info.py:probe():152] Probing system
+2024-09-01 08:08:30,711 DEBUG   HandlerThread:1314 [system_info.py:_probe_git():137] Probing git
+2024-09-01 08:08:30,718 DEBUG   HandlerThread:1314 [system_info.py:_probe_git():145] Probing git done
+2024-09-01 08:08:30,718 DEBUG   HandlerThread:1314 [system_info.py:probe():200] Probing system done
+2024-09-01 08:08:30,718 DEBUG   HandlerThread:1314 [system_monitor.py:probe():223] {'os': 'Linux-6.5.0-35-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-09-01T08:08:30.709894', 'startedAt': '2024-09-01T08:08:30.425931', 'docker': None, 'cuda': None, 'args': ('example.yaml',), 'state': 'running', 'program': '-m train', 'codePathLocal': None, 'git': {'remote': 'https://github.com/mistralai/mistral-finetune.git', 'commit': '0b0eaac415ba6317e198aa2207c7bd4b3973adbe'}, 'email': None, 'root': '/workspace/mistral-finetune', 'host': 'f6e183d76cb1', 'username': 'root', 'executable': '/usr/bin/python', 'cpu_count': 32, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1976.101125, 'min': 1500.0, 'max': 3200.0}, 'cpu_freq_per_core': [{'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3925.001, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 2773.255, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 2221.478, 'min': 1500.0, 'max': 3200.0}, {'current': 2062.966, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 2111.787, 'min': 1500.0, 'max': 3200.0}, {'current': 3925.001, 'min': 1500.0, 'max': 3200.0}, {'current': 3137.781, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1499.733, 'min': 1500.0, 'max': 3200.0}, {'current': 2276.722, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1606.878, 'min': 1500.0, 'max': 3200.0}, {'current': 3120.917, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3924.981, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1916.093, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3924.985, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1798.162, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3126.511, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3139.064, 'min': 1500.0, 'max': 3200.0}, {'current': 3137.864, 'min': 1500.0, 'max': 3200.0}], 'disk': {'/': {'total': 50.0, 'used': 4.066852569580078}}, 'gpu': 'NVIDIA RTX A6000', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA RTX A6000', 'memory_total': 51527024640}], 'memory': {'total': 503.7315444946289}}
+2024-09-01 08:08:30,718 INFO    HandlerThread:1314 [system_monitor.py:probe():224] Finished collecting system info
+2024-09-01 08:08:30,718 INFO    HandlerThread:1314 [system_monitor.py:probe():227] Publishing system info
+2024-09-01 08:08:30,719 INFO    HandlerThread:1314 [system_monitor.py:probe():229] Finished publishing system info
+2024-09-01 08:08:30,724 DEBUG   SenderThread:1314 [sender.py:send():391] send: files
+2024-09-01 08:08:30,724 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-metadata.json with policy now
+2024-09-01 08:08:30,806 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: python_packages
+2024-09-01 08:08:30,806 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: python_packages
+2024-09-01 08:08:30,807 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-01 08:08:30,808 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: stop_status
+2024-09-01 08:08:30,809 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:08:31,023 DEBUG   SenderThread:1314 [sender.py:send():391] send: telemetry
+2024-09-01 08:08:31,143 INFO    wandb-upload_0:1314 [upload_job.py:push():130] Uploaded file /tmp/tmp53jjcldxwandb/h8od4rgj-wandb-metadata.json
+2024-09-01 08:08:31,662 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_created():271] file/dir created: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-metadata.json
+2024-09-01 08:08:31,662 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_created():271] file/dir created: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:31,662 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_created():271] file/dir created: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/requirements.txt
+2024-09-01 08:08:33,662 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:35,486 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:37,663 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:39,664 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:40,093 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:40,094 DEBUG   SenderThread:1314 [sender.py:send():391] send: telemetry
+2024-09-01 08:08:40,800 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:40,813 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:08:41,665 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:42,978 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:42,979 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:42,979 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:42,980 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:43,666 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:43,666 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_created():271] file/dir created: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:45,667 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:45,808 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-01 08:08:45,808 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: stop_status
+2024-09-01 08:08:45,879 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:45,954 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:45,955 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:45,955 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:45,956 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:46,667 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:47,667 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:48,830 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:48,831 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:48,832 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:48,832 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:49,668 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:49,669 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:50,809 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:08:50,894 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:51,669 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:52,333 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:52,334 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:52,334 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:52,334 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:52,335 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:52,670 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:53,670 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:55,120 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:55,121 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:55,122 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:55,122 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:55,671 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:55,672 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:56,810 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:57,672 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:58,009 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:58,010 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:58,011 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:58,012 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:58,673 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:59,674 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:00,808 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-01 08:09:00,809 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:09:00,810 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: stop_status
+2024-09-01 08:09:00,953 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:09:00,954 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:09:00,955 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:09:00,955 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:01,674 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:01,675 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:02,613 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:03,675 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/config.yaml
+2024-09-01 08:09:03,676 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:03,815 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:09:03,817 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:09:03,817 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:09:03,817 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:04,676 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:05,676 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:07,247 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:09:07,248 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:09:07,249 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:09:07,249 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:09:07,250 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:07,678 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:07,678 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:08,251 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:09,678 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:10,825 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:09:13,826 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:15,810 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-01 08:09:15,811 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: stop_status
+2024-09-01 08:09:18,980 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:20,837 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:09:24,838 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:29,702 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: check_version
+2024-09-01 08:09:29,703 DEBUG   SenderThread:1314 [sender.py:send():391] send: telemetry
+2024-09-01 08:09:29,703 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: check_version
+2024-09-01 08:09:29,803 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: server_info
+2024-09-01 08:09:29,803 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: server_info
+2024-09-01 08:09:29,870 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:29,871 DEBUG   SenderThread:1314 [sender.py:send():391] send: exit
+2024-09-01 08:09:29,872 INFO    SenderThread:1314 [sender.py:send_exit():598] handling exit code: 0
+2024-09-01 08:09:29,872 INFO    SenderThread:1314 [sender.py:send_exit():600] handling runtime: 59
+2024-09-01 08:09:29,874 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:29,875 INFO    SenderThread:1314 [sender.py:send_exit():606] send defer
+2024-09-01 08:09:29,875 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,875 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 0
+2024-09-01 08:09:29,875 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,875 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 0
+2024-09-01 08:09:29,876 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 1
+2024-09-01 08:09:29,876 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,876 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 1
+2024-09-01 08:09:29,876 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,876 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 1
+2024-09-01 08:09:29,876 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 2
+2024-09-01 08:09:29,876 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,877 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 2
+2024-09-01 08:09:29,877 INFO    HandlerThread:1314 [system_monitor.py:finish():203] Stopping system monitor
+2024-09-01 08:09:29,877 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined cpu monitor
+2024-09-01 08:09:29,878 DEBUG   SystemMonitor:1314 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-09-01 08:09:29,878 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined disk monitor
+2024-09-01 08:09:29,878 DEBUG   SystemMonitor:1314 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-09-01 08:09:29,879 DEBUG   SystemMonitor:1314 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-09-01 08:09:29,895 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined gpu monitor
+2024-09-01 08:09:29,895 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined memory monitor
+2024-09-01 08:09:29,896 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined network monitor
+2024-09-01 08:09:29,896 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,896 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 2
+2024-09-01 08:09:29,896 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 3
+2024-09-01 08:09:29,896 DEBUG   SenderThread:1314 [sender.py:send():391] send: stats
+2024-09-01 08:09:29,897 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,897 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 3
+2024-09-01 08:09:29,899 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:09:29,899 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:09:29,899 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:29,900 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,900 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 3
+2024-09-01 08:09:29,900 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 4
+2024-09-01 08:09:29,900 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,900 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 4
+2024-09-01 08:09:29,901 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,901 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 4
+2024-09-01 08:09:29,901 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 5
+2024-09-01 08:09:29,901 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,901 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 5
+2024-09-01 08:09:29,901 DEBUG   SenderThread:1314 [sender.py:send():391] send: summary
+2024-09-01 08:09:29,902 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:29,902 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,902 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 5
+2024-09-01 08:09:29,902 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 6
+2024-09-01 08:09:29,902 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,903 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 6
+2024-09-01 08:09:29,903 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,903 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 6
+2024-09-01 08:09:29,908 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:30,006 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 7
+2024-09-01 08:09:30,006 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:30,006 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 7
+2024-09-01 08:09:30,006 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:30,007 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 7
+2024-09-01 08:09:30,687 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/config.yaml
+2024-09-01 08:09:30,687 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:30,872 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: poll_exit
+2024-09-01 08:09:31,156 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 8
+2024-09-01 08:09:31,157 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: poll_exit
+2024-09-01 08:09:31,157 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:31,157 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 8
+2024-09-01 08:09:31,158 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:31,158 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 8
+2024-09-01 08:09:31,158 INFO    SenderThread:1314 [job_builder.py:build():443] Attempting to build job artifact
+2024-09-01 08:09:31,159 INFO    SenderThread:1314 [job_builder.py:_get_source_type():572] is repo sourced job
+2024-09-01 08:09:31,159 WARNING SenderThread:1314 [job_builder.py:_log_if_verbose():274] No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job
+2024-09-01 08:09:31,159 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 9
+2024-09-01 08:09:31,159 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:31,159 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 9
+2024-09-01 08:09:31,159 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:31,160 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 9
+2024-09-01 08:09:31,160 INFO    SenderThread:1314 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-09-01 08:09:31,687 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:31,688 INFO    SenderThread:1314 [dir_watcher.py:finish():388] scan: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files
+2024-09-01 08:09:31,688 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/config.yaml config.yaml
+2024-09-01 08:09:31,688 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-metadata.json wandb-metadata.json
+2024-09-01 08:09:31,688 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/requirements.txt requirements.txt
+2024-09-01 08:09:31,691 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log output.log
+2024-09-01 08:09:31,694 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json wandb-summary.json
+2024-09-01 08:09:31,696 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 10
+2024-09-01 08:09:31,696 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:31,698 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 10
+2024-09-01 08:09:31,698 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:31,698 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 10
+2024-09-01 08:09:31,698 INFO    SenderThread:1314 [file_pusher.py:finish():169] shutting down file pusher
+2024-09-01 08:09:31,872 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: poll_exit
+2024-09-01 08:09:31,873 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: poll_exit
+2024-09-01 08:09:31,928 INFO    wandb-upload_0:1314 [upload_job.py:push():130] Uploaded file content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/config.yaml
+2024-09-01 08:09:32,038 INFO    wandb-upload_1:1314 [upload_job.py:push():130] Uploaded file content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/requirements.txt
+2024-09-01 08:09:32,135 INFO    wandb-upload_3:1314 [upload_job.py:push():130] Uploaded file content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:32,170 INFO    wandb-upload_2:1314 [upload_job.py:push():130] Uploaded file content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:32,370 INFO    Thread-11 (_thread_body):1314 [sender.py:transition_state():626] send defer: 11
+2024-09-01 08:09:32,370 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:32,370 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 11
+2024-09-01 08:09:32,371 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:32,371 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 11
+2024-09-01 08:09:32,371 INFO    SenderThread:1314 [file_pusher.py:join():175] waiting for file pusher
+2024-09-01 08:09:32,372 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 12
+2024-09-01 08:09:32,372 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:32,372 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 12
+2024-09-01 08:09:32,372 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:32,373 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 12
+2024-09-01 08:09:32,373 INFO    SenderThread:1314 [file_stream.py:finish():601] file stream finish called
+2024-09-01 08:09:32,612 INFO    SenderThread:1314 [file_stream.py:finish():605] file stream finish is done
+2024-09-01 08:09:32,612 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 13
+2024-09-01 08:09:32,612 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:32,612 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 13
+2024-09-01 08:09:32,613 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:32,613 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 13
+2024-09-01 08:09:32,613 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 14
+2024-09-01 08:09:32,613 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:32,613 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 14
+2024-09-01 08:09:32,613 DEBUG   SenderThread:1314 [sender.py:send():391] send: final
+2024-09-01 08:09:32,614 DEBUG   SenderThread:1314 [sender.py:send():391] send: footer
+2024-09-01 08:09:32,614 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:32,614 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 14
+2024-09-01 08:09:32,615 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: poll_exit
+2024-09-01 08:09:32,615 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: poll_exit
+2024-09-01 08:09:32,616 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:09:32,616 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: get_summary
+2024-09-01 08:09:32,617 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: sampled_history
+2024-09-01 08:09:32,618 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: shutdown
+2024-09-01 08:09:32,618 INFO    HandlerThread:1314 [handler.py:finish():884] shutting down handler
+2024-09-01 08:09:33,615 INFO    WriterThread:1314 [datastore.py:close():296] close: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/run-ra4ttuu9.wandb
+2024-09-01 08:09:33,615 INFO    SenderThread:1314 [sender.py:finish():1631] shutting down sender
+2024-09-01 08:09:33,616 INFO    SenderThread:1314 [file_pusher.py:finish():169] shutting down file pusher
+2024-09-01 08:09:33,616 INFO    SenderThread:1314 [file_pusher.py:join():175] waiting for file pusher

wandb/debug.log

@@ -0,0 +1,36 @@
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Current SDK version is 0.17.8
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Configure stats pid to 1234
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Loading settings from /root/.config/wandb/settings
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Loading settings from /workspace/mistral-finetune/wandb/settings
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Loading settings from environment variables: {}
+2024-09-01 08:08:30,428 WARNING MainThread:1234 [wandb_setup.py:_flush():77] Could not find program at -m train
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m train'}
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Applying login settings: {'api_key': '***REDACTED***'}
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Applying login settings: {'api_key': '***REDACTED***'}
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Applying login settings: {}
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_init.py:_log_setup():524] Logging user logs to content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/logs/debug.log
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_init.py:_log_setup():525] Logging internal logs to content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/logs/debug-internal.log
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_init.py:init():607] calling init triggers
+2024-09-01 08:08:30,429 INFO    MainThread:1234 [wandb_init.py:init():614] wandb.init called with sweep_config: {}
+config: {'data': {'data': '', 'shuffle': False, 'instruct_data': './data/t_train.jsonl', 'eval_instruct_data': './data/t_eval.jsonl', 'instruct': {'shuffle': True, 'dynamic_chunk_fn_call': True}}, 'model_id_or_path': '../downloaded_model/mistral_models/7B-v0.3/', 'run_dir': './content/test_ultra_1', 'optim': {'lr': 0.0001, 'weight_decay': 0.1, 'pct_start': 0.05}, 'seed': 0, 'num_microbatches': 4, 'seq_len': 1024, 'batch_size': 1, 'max_norm': 1.0, 'max_steps': 10, 'log_freq': 1, 'ckpt_freq': 10, 'save_adapters': False, 'no_ckpt': False, 'num_ckpt_keep': 3, 'eval_freq': 5, 'no_eval': False, 'checkpoint': True, 'world_size': 1, 'wandb': {'project': 'finetuning_mistral', 'offline': False, 'key': '9e930bac040e715c4be9de064318956067b45479', 'run_name': None}, 'mlflow': {'tracking_uri': None, 'experiment_name': None}, 'lora': {'enable': True, 'rank': 16, 'dropout': 0.05, 'scaling': 2.0}}
+2024-09-01 08:08:30,429 INFO    MainThread:1234 [wandb_init.py:init():657] starting backend
+2024-09-01 08:08:30,429 INFO    MainThread:1234 [wandb_init.py:init():661] setting up manager
+2024-09-01 08:08:30,431 INFO    MainThread:1234 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-09-01 08:08:30,432 INFO    MainThread:1234 [wandb_init.py:init():669] backend started and connected
+2024-09-01 08:08:30,436 INFO    MainThread:1234 [wandb_init.py:init():767] updated telemetry
+2024-09-01 08:08:30,441 INFO    MainThread:1234 [wandb_init.py:init():800] communicating run to backend with 90.0 second timeout
+2024-09-01 08:08:30,665 INFO    MainThread:1234 [wandb_init.py:init():851] starting run threads in backend
+2024-09-01 08:08:30,808 INFO    MainThread:1234 [wandb_run.py:_console_start():2463] atexit reg
+2024-09-01 08:08:30,808 INFO    MainThread:1234 [wandb_run.py:_redirect():2309] redirect: wrap_raw
+2024-09-01 08:08:30,808 INFO    MainThread:1234 [wandb_run.py:_redirect():2374] Wrapping output streams.
+2024-09-01 08:08:30,808 INFO    MainThread:1234 [wandb_run.py:_redirect():2399] Redirects installed.
+2024-09-01 08:08:30,810 INFO    MainThread:1234 [wandb_init.py:init():894] run started, returning control to user process
+2024-09-01 08:09:29,670 INFO    MainThread:1234 [wandb_run.py:_finish():2160] finishing run dwivedi-rishabh95-avizva/finetuning_mistral/ra4ttuu9
+2024-09-01 08:09:29,671 INFO    MainThread:1234 [wandb_run.py:_atexit_cleanup():2424] got exitcode: 0
+2024-09-01 08:09:29,671 INFO    MainThread:1234 [wandb_run.py:_restore():2406] restore
+2024-09-01 08:09:29,671 INFO    MainThread:1234 [wandb_run.py:_restore():2412] restore done
+2024-09-01 08:09:29,671 INFO    MainThread:1234 [wandb_run.py:_on_finish():2677] communicating current version
+2024-09-01 08:09:29,802 INFO    MainThread:1234 [wandb_run.py:_on_finish():2686] got version response 
+2024-09-01 08:09:33,620 INFO    MainThread:1234 [wandb_run.py:_footer_history_summary_info():4078] rendering history
+2024-09-01 08:09:33,620 INFO    MainThread:1234 [wandb_run.py:_footer_history_summary_info():4110] rendering summary
+2024-09-01 08:09:33,629 INFO    MainThread:1234 [wandb_run.py:_footer_sync_info():4037] logging synced files

wandb/run-20240901_080830-ra4ttuu9/files/config.yaml

@@ -0,0 +1,115 @@
+wandb_version: 1
+
+data:
+  desc: null
+  value:
+    data: ''
+    shuffle: false
+    instruct_data: ./data/t_train.jsonl
+    eval_instruct_data: ./data/t_eval.jsonl
+    instruct:
+      shuffle: true
+      dynamic_chunk_fn_call: true
+model_id_or_path:
+  desc: null
+  value: ../downloaded_model/mistral_models/7B-v0.3/
+run_dir:
+  desc: null
+  value: ./content/test_ultra_1
+optim:
+  desc: null
+  value:
+    lr: 0.0001
+    weight_decay: 0.1
+    pct_start: 0.05
+seed:
+  desc: null
+  value: 0
+num_microbatches:
+  desc: null
+  value: 4
+seq_len:
+  desc: null
+  value: 1024
+batch_size:
+  desc: null
+  value: 1
+max_norm:
+  desc: null
+  value: 1.0
+max_steps:
+  desc: null
+  value: 10
+log_freq:
+  desc: null
+  value: 1
+ckpt_freq:
+  desc: null
+  value: 10
+save_adapters:
+  desc: null
+  value: false
+no_ckpt:
+  desc: null
+  value: false
+num_ckpt_keep:
+  desc: null
+  value: 3
+eval_freq:
+  desc: null
+  value: 5
+no_eval:
+  desc: null
+  value: false
+checkpoint:
+  desc: null
+  value: true
+world_size:
+  desc: null
+  value: 1
+wandb:
+  desc: null
+  value:
+    project: finetuning_mistral
+    offline: false
+    key: 9e930bac040e715c4be9de064318956067b45479
+    run_name: null
+mlflow:
+  desc: null
+  value:
+    tracking_uri: null
+    experiment_name: null
+lora:
+  desc: null
+  value:
+    enable: true
+    rank: 16
+    dropout: 0.05
+    scaling: 2.0
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.17.8
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1725178110
+    t:
+      1:
+      - 1
+      - 55
+      2:
+      - 1
+      - 55
+      3:
+      - 2
+      - 13
+      - 16
+      - 23
+      - 61
+      4: 3.10.12
+      5: 0.17.8
+      8:
+      - 5
+      13: linux-x86_64

wandb/run-20240901_080830-ra4ttuu9/files/output.log

@@ -0,0 +1,55 @@
+
+wandb: WARNING Calling wandb.login() after wandb.init() has no effect.
+2024-09-01 08:08:30 (UTC) - 0:00:04 - finetune.wrapped_model - INFO - Reloading model from ../downloaded_model/mistral_models/7B-v0.3/consolidated.safetensors ...
+2024-09-01 08:08:30 (UTC) - 0:00:04 - finetune.wrapped_model - INFO - Converting model to dtype torch.bfloat16 ...
+2024-09-01 08:08:30 (UTC) - 0:00:04 - finetune.wrapped_model - INFO - Loaded model on cpu!
+2024-09-01 08:08:30 (UTC) - 0:00:04 - finetune.wrapped_model - INFO - Initializing lora layers ...
+2024-09-01 08:08:32 (UTC) - 0:00:05 - finetune.wrapped_model - INFO - Finished initialization!
+2024-09-01 08:08:32 (UTC) - 0:00:05 - finetune.wrapped_model - INFO - Sharding model over 1 GPUs ...
+2024-09-01 08:08:35 (UTC) - 0:00:08 - finetune.wrapped_model - INFO - Model sharded!
+2024-09-01 08:08:35 (UTC) - 0:00:08 - finetune.wrapped_model - INFO - 41,943,040 out of 7,289,966,592 parameters are finetuned (0.58%).
+2024-09-01 08:08:36 (UTC) - 0:00:09 - dataset - INFO - Loading data/t_train.jsonl ...
+2024-09-01 08:08:36 (UTC) - 0:00:09 - dataset - INFO - data/t_train.jsonl loaded and tokenized.
+2024-09-01 08:08:36 (UTC) - 0:00:09 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:38 (UTC) - 0:00:11 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:40 (UTC) - 0:00:13 - train - INFO - step: 000001 - done (%): 10.0 - loss: 1.135 - lr: 9.9e-05 - peak_alloc_mem (GB): 14.7 - alloc_mem (GB): 14.3 - words_per_second: 1177.8 - avg_words_per_second: 1177.8 - ETA: >2024-09-01 08:09:11
+2024-09-01 08:08:40 (UTC) - 0:00:13 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:40 (UTC) - 0:00:14 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:42 (UTC) - 0:00:15 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:42 (UTC) - 0:00:16 - train - INFO - step: 000002 - done (%): 20.0 - loss: 0.882 - lr: 9.4e-05 - peak_alloc_mem (GB): 15.0 - alloc_mem (GB): 14.3 - words_per_second: 1420.5 - avg_words_per_second: 1287.8 - ETA: >2024-09-01 08:09:08
+2024-09-01 08:08:43 (UTC) - 0:00:17 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:45 (UTC) - 0:00:18 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:45 (UTC) - 0:00:19 - train - INFO - step: 000003 - done (%): 30.0 - loss: 0.757 - lr: 8.4e-05 - peak_alloc_mem (GB): 15.0 - alloc_mem (GB): 14.3 - words_per_second: 1376.9 - avg_words_per_second: 1316.2 - ETA: >2024-09-01 08:09:07
+2024-09-01 08:08:46 (UTC) - 0:00:19 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:48 (UTC) - 0:00:21 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:48 (UTC) - 0:00:22 - train - INFO - step: 000004 - done (%): 40.0 - loss: 0.465 - lr: 7.0e-05 - peak_alloc_mem (GB): 15.0 - alloc_mem (GB): 14.3 - words_per_second: 1424.8 - avg_words_per_second: 1341.8 - ETA: >2024-09-01 08:09:07
+2024-09-01 08:08:49 (UTC) - 0:00:22 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:50 (UTC) - 0:00:24 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:51 (UTC) - 0:00:24 - eval - INFO - Start eval...
+2024-09-01 08:08:52 (UTC) - 0:00:25 - eval - INFO - Eval finished!
+2024-09-01 08:08:52 (UTC) - 0:00:25 - train - INFO - step: 000005 - eval_perplexity: 1.260 - eval_loss: 0.334 - train_loss: 0.306
+2024-09-01 08:08:52 (UTC) - 0:00:25 - train - INFO - step: 000005 - done (%): 50.0 - loss: 0.306 - lr: 5.4e-05 - peak_alloc_mem (GB): 15.0 - alloc_mem (GB): 14.3 - words_per_second: 1169.4 - avg_words_per_second: 1303.4 - ETA: >2024-09-01 08:09:08
+2024-09-01 08:08:53 (UTC) - 0:00:26 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:54 (UTC) - 0:00:27 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:55 (UTC) - 0:00:28 - train - INFO - step: 000006 - done (%): 60.0 - loss: 0.225 - lr: 3.8e-05 - peak_alloc_mem (GB): 15.0 - alloc_mem (GB): 14.3 - words_per_second: 1470.7 - avg_words_per_second: 1328.6 - ETA: >2024-09-01 08:09:07
+2024-09-01 08:08:55 (UTC) - 0:00:29 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:57 (UTC) - 0:00:30 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:08:58 (UTC) - 0:00:31 - train - INFO - step: 000007 - done (%): 70.0 - loss: 0.164 - lr: 2.3e-05 - peak_alloc_mem (GB): 15.0 - alloc_mem (GB): 14.3 - words_per_second: 1418.2 - avg_words_per_second: 1340.7 - ETA: >2024-09-01 08:09:07
+2024-09-01 08:08:58 (UTC) - 0:00:31 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:09:00 (UTC) - 0:00:33 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:09:00 (UTC) - 0:00:34 - train - INFO - step: 000008 - done (%): 80.0 - loss: 0.130 - lr: 1.1e-05 - peak_alloc_mem (GB): 15.0 - alloc_mem (GB): 14.3 - words_per_second: 1411.1 - avg_words_per_second: 1349.1 - ETA: >2024-09-01 08:09:06
+2024-09-01 08:09:01 (UTC) - 0:00:34 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:09:02 (UTC) - 0:00:36 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:09:03 (UTC) - 0:00:37 - train - INFO - step: 000009 - done (%): 90.0 - loss: 0.152 - lr: 2.7e-06 - peak_alloc_mem (GB): 15.0 - alloc_mem (GB): 14.3 - words_per_second: 1412.0 - avg_words_per_second: 1355.8 - ETA: >2024-09-01 08:09:06
+2024-09-01 08:09:04 (UTC) - 0:00:37 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:09:05 (UTC) - 0:00:39 - dataset - INFO - Shuffling data/t_train.jsonl ...
+2024-09-01 08:09:06 (UTC) - 0:00:39 - eval - INFO - Start eval...
+2024-09-01 08:09:07 (UTC) - 0:00:40 - eval - INFO - Eval finished!
+2024-09-01 08:09:07 (UTC) - 0:00:40 - train - INFO - step: 000010 - eval_perplexity: 1.134 - eval_loss: 0.182 - train_loss: 0.108
+2024-09-01 08:09:07 (UTC) - 0:00:40 - train - INFO - step: 000010 - done (%): 100.0 - loss: 0.108 - lr: 4.0e-10 - peak_alloc_mem (GB): 15.0 - alloc_mem (GB): 14.3 - words_per_second: 1193.5 - avg_words_per_second: 1337.6 - ETA: >2024-09-01 08:09:07
+2024-09-01 08:09:07 (UTC) - 0:00:40 - checkpointing - INFO - Dumping checkpoint in content/test_ultra_1/checkpoints/checkpoint_000010/consolidated using tmp name: tmp.consolidated
+2024-09-01 08:09:29 (UTC) - 0:01:02 - checkpointing - INFO - Done dumping checkpoint in content/test_ultra_1/checkpoints/checkpoint_000010/consolidated for step: 10
+2024-09-01 08:09:29 (UTC) - 0:01:02 - checkpointing - INFO - Done deleting checkpoints
+2024-09-01 08:09:29 (UTC) - 0:01:02 - checkpointing - INFO - Done!
+2024-09-01 08:09:29 (UTC) - 0:01:02 - train - INFO - done!
+2024-09-01 08:09:29 (UTC) - 0:01:02 - utils - INFO - Closing: eval_logger

wandb/run-20240901_080830-ra4ttuu9/files/requirements.txt

@@ -0,0 +1,182 @@
+Babel==2.13.1
+GitPython==3.1.43
+Jinja2==3.1.2
+Markdown==3.7
+MarkupSafe==2.1.3
+Pillow==10.1.0
+PyGObject==3.42.1
+PyJWT==2.3.0
+PyYAML==6.0.1
+Pygments==2.17.2
+SecretStorage==3.3.1
+Send2Trash==1.8.2
+Werkzeug==3.0.4
+absl-py==2.1.0
+annotated-types==0.7.0
+anyio==4.1.0
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+asttokens==2.4.1
+async-lru==2.0.4
+attrs==23.1.0
+beautifulsoup4==4.12.2
+bleach==6.1.0
+blinker==1.4
+certifi==2023.11.17
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+comm==0.2.0
+cryptography==3.4.8
+dbus-python==1.2.18
+debugpy==1.8.0
+decorator==5.1.1
+defusedxml==0.7.1
+distro==1.7.0
+docker-pycreds==0.4.0
+docstring_parser==0.16
+entrypoints==0.4
+et-xmlfile==1.1.0
+exceptiongroup==1.2.0
+executing==2.0.1
+fastjsonschema==2.19.0
+filelock==3.13.1
+fire==0.6.0
+fqdn==1.5.1
+fsspec==2023.10.0
+gitdb==4.0.11
+grpcio==1.66.1
+httplib2==0.20.2
+huggingface-hub==0.24.6
+idna==3.6
+importlib-metadata==4.6.4
+ipykernel==6.26.0
+ipython-genutils==0.2.0
+ipython==8.18.1
+ipywidgets==8.1.1
+isoduration==20.11.0
+jedi==0.19.1
+jeepney==0.7.1
+json5==0.9.14
+jsonpointer==2.4
+jsonschema-specifications==2023.11.1
+jsonschema==4.23.0
+jupyter-archive==3.4.0
+jupyter-contrib-core==0.4.2
+jupyter-contrib-nbextensions==0.7.0
+jupyter-events==0.9.0
+jupyter-highlight-selected-word==0.2.0
+jupyter-lsp==2.2.1
+jupyter-nbextensions-configurator==0.6.3
+jupyter_client==7.4.9
+jupyter_core==5.5.0
+jupyter_server==2.10.1
+jupyter_server_terminals==0.4.4
+jupyterlab-widgets==3.0.9
+jupyterlab==4.0.9
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.25.2
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+lxml==4.9.3
+matplotlib-inline==0.1.6
+mistral_common==1.3.4
+mistune==3.0.2
+more-itertools==8.10.0
+mpmath==1.3.0
+nbclassic==1.0.0
+nbclient==0.9.0
+nbconvert==7.11.0
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.2.1
+notebook==6.5.5
+notebook_shim==0.2.3
+numpy==1.26.2
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.19.3
+nvidia-nvjitlink-cu12==12.3.101
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.0
+openpyxl==3.1.5
+overrides==7.4.0
+packaging==23.2
+pandas==2.2.2
+pandocfilters==1.5.0
+parso==0.8.3
+pexpect==4.9.0
+pip==23.3.1
+platformdirs==4.0.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.41
+protobuf==5.28.0
+psutil==5.9.6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pycparser==2.21
+pydantic==2.8.2
+pydantic_core==2.20.1
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu2
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+pytz==2024.1
+pyzmq==24.0.1
+referencing==0.31.0
+regex==2024.7.24
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.13.1
+safetensors==0.4.4
+sentencepiece==0.2.0
+sentry-sdk==2.13.0
+setproctitle==1.3.3
+setuptools==69.0.2
+simple_parsing==0.1.5
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+soupsieve==2.5
+stack-data==0.6.3
+sympy==1.12
+tensorboard-data-server==0.7.2
+tensorboard==2.17.1
+termcolor==2.4.0
+terminado==0.18.0
+tiktoken==0.7.0
+tinycss2==1.2.1
+tomli==2.0.1
+torch==2.2.0
+torchaudio==2.1.1
+torchvision==0.16.1
+tornado==6.3.3
+tqdm==4.66.5
+traitlets==5.13.0
+triton==2.2.0
+types-python-dateutil==2.8.19.14
+typing_extensions==4.12.2
+tzdata==2024.1
+uri-template==1.3.0
+urllib3==2.1.0
+wadllib==1.3.6
+wandb==0.17.8
+wcwidth==0.2.12
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.6.4
+wheel==0.42.0
+widgetsnbextension==4.0.9
+xformers==0.0.24
+zipp==1.0.0

wandb/run-20240901_080830-ra4ttuu9/files/wandb-metadata.json

@@ -0,0 +1,369 @@
+{
+    "os": "Linux-6.5.0-35-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-09-01T08:08:30.709894",
+    "startedAt": "2024-09-01T08:08:30.425931",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "example.yaml"
+    ],
+    "state": "running",
+    "program": "-m train",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://github.com/mistralai/mistral-finetune.git",
+        "commit": "0b0eaac415ba6317e198aa2207c7bd4b3973adbe"
+    },
+    "email": null,
+    "root": "/workspace/mistral-finetune",
+    "host": "f6e183d76cb1",
+    "username": "root",
+    "executable": "/usr/bin/python",
+    "cpu_count": 32,
+    "cpu_count_logical": 64,
+    "cpu_freq": {
+        "current": 1976.101125,
+        "min": 1500.0,
+        "max": 3200.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3925.001,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3200.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 2773.255,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3200.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 2221.478,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 2062.966,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3200.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 2111.787,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3925.001,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3137.781,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3200.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1499.733,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 2276.722,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1606.878,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3120.917,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3924.981,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1916.093,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3924.985,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3200.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1798.162,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3200.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3126.511,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3200.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3139.064,
+            "min": 1500.0,
+            "max": 3200.0
+        },
+        {
+            "current": 3137.864,
+            "min": 1500.0,
+            "max": 3200.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 50.0,
+            "used": 4.066852569580078
+        }
+    },
+    "gpu": "NVIDIA RTX A6000",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA RTX A6000",
+            "memory_total": 51527024640
+        }
+    ],
+    "memory": {
+        "total": 503.7315444946289
+    }
+}

wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/lr": 3.9999999999999996e-10, "train/loss": 0.108039490878582, "train/percent_done": 100.0, "train/peak_allocated_mem": 14.968535900115967, "train/allocated_mem": 14.327418327331543, "train/wps": 1193.533250331889, "train/avg_wps": 1337.6038764958669, "train/eta_in_seconds": 0.0, "_timestamp": 1725178147.2484581, "_runtime": 36.81593322753906, "_step": 10, "eval/train_loss": 0.108039490878582, "eval/perplexity": 1.1342318058013916, "eval/eval_loss": 0.18171557784080505, "_wandb": {"runtime": 59}}

wandb/run-20240901_080830-ra4ttuu9/logs/debug-internal.log

@@ -0,0 +1,266 @@
+2024-09-01 08:08:30,433 INFO    StreamThr :1314 [internal.py:wandb_internal():85] W&B internal server running at pid: 1314, started at: 2024-09-01 08:08:30.432847
+2024-09-01 08:08:30,434 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status
+2024-09-01 08:08:30,436 INFO    WriterThread:1314 [datastore.py:open_for_write():87] open: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/run-ra4ttuu9.wandb
+2024-09-01 08:08:30,437 DEBUG   SenderThread:1314 [sender.py:send():391] send: header
+2024-09-01 08:08:30,442 DEBUG   SenderThread:1314 [sender.py:send():391] send: run
+2024-09-01 08:08:30,661 INFO    SenderThread:1314 [dir_watcher.py:__init__():211] watching files in: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files
+2024-09-01 08:08:30,661 INFO    SenderThread:1314 [sender.py:_start_run_threads():1200] run started: ra4ttuu9 with start time 1725178110.432525
+2024-09-01 08:08:30,668 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: run_start
+2024-09-01 08:08:30,682 DEBUG   HandlerThread:1314 [system_info.py:__init__():26] System info init
+2024-09-01 08:08:30,682 DEBUG   HandlerThread:1314 [system_info.py:__init__():41] System info init done
+2024-09-01 08:08:30,682 INFO    HandlerThread:1314 [system_monitor.py:start():194] Starting system monitor
+2024-09-01 08:08:30,683 INFO    SystemMonitor:1314 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-09-01 08:08:30,683 INFO    HandlerThread:1314 [system_monitor.py:probe():214] Collecting system info
+2024-09-01 08:08:30,684 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started cpu monitoring
+2024-09-01 08:08:30,684 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started disk monitoring
+2024-09-01 08:08:30,685 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started gpu monitoring
+2024-09-01 08:08:30,686 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started memory monitoring
+2024-09-01 08:08:30,686 INFO    SystemMonitor:1314 [interfaces.py:start():188] Started network monitoring
+2024-09-01 08:08:30,709 DEBUG   HandlerThread:1314 [system_info.py:probe():152] Probing system
+2024-09-01 08:08:30,711 DEBUG   HandlerThread:1314 [system_info.py:_probe_git():137] Probing git
+2024-09-01 08:08:30,718 DEBUG   HandlerThread:1314 [system_info.py:_probe_git():145] Probing git done
+2024-09-01 08:08:30,718 DEBUG   HandlerThread:1314 [system_info.py:probe():200] Probing system done
+2024-09-01 08:08:30,718 DEBUG   HandlerThread:1314 [system_monitor.py:probe():223] {'os': 'Linux-6.5.0-35-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-09-01T08:08:30.709894', 'startedAt': '2024-09-01T08:08:30.425931', 'docker': None, 'cuda': None, 'args': ('example.yaml',), 'state': 'running', 'program': '-m train', 'codePathLocal': None, 'git': {'remote': 'https://github.com/mistralai/mistral-finetune.git', 'commit': '0b0eaac415ba6317e198aa2207c7bd4b3973adbe'}, 'email': None, 'root': '/workspace/mistral-finetune', 'host': 'f6e183d76cb1', 'username': 'root', 'executable': '/usr/bin/python', 'cpu_count': 32, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1976.101125, 'min': 1500.0, 'max': 3200.0}, 'cpu_freq_per_core': [{'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3925.001, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 2773.255, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 2221.478, 'min': 1500.0, 'max': 3200.0}, {'current': 2062.966, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 2111.787, 'min': 1500.0, 'max': 3200.0}, {'current': 3925.001, 'min': 1500.0, 'max': 3200.0}, {'current': 3137.781, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1499.733, 'min': 1500.0, 'max': 3200.0}, {'current': 2276.722, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1606.878, 'min': 1500.0, 'max': 3200.0}, {'current': 3120.917, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3924.981, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1916.093, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3924.985, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1798.162, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3126.511, 'min': 1500.0, 'max': 3200.0}, {'current': 3200.0, 'min': 1500.0, 'max': 3200.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3200.0}, {'current': 3139.064, 'min': 1500.0, 'max': 3200.0}, {'current': 3137.864, 'min': 1500.0, 'max': 3200.0}], 'disk': {'/': {'total': 50.0, 'used': 4.066852569580078}}, 'gpu': 'NVIDIA RTX A6000', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA RTX A6000', 'memory_total': 51527024640}], 'memory': {'total': 503.7315444946289}}
+2024-09-01 08:08:30,718 INFO    HandlerThread:1314 [system_monitor.py:probe():224] Finished collecting system info
+2024-09-01 08:08:30,718 INFO    HandlerThread:1314 [system_monitor.py:probe():227] Publishing system info
+2024-09-01 08:08:30,719 INFO    HandlerThread:1314 [system_monitor.py:probe():229] Finished publishing system info
+2024-09-01 08:08:30,724 DEBUG   SenderThread:1314 [sender.py:send():391] send: files
+2024-09-01 08:08:30,724 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-metadata.json with policy now
+2024-09-01 08:08:30,806 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: python_packages
+2024-09-01 08:08:30,806 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: python_packages
+2024-09-01 08:08:30,807 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-01 08:08:30,808 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: stop_status
+2024-09-01 08:08:30,809 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:08:31,023 DEBUG   SenderThread:1314 [sender.py:send():391] send: telemetry
+2024-09-01 08:08:31,143 INFO    wandb-upload_0:1314 [upload_job.py:push():130] Uploaded file /tmp/tmp53jjcldxwandb/h8od4rgj-wandb-metadata.json
+2024-09-01 08:08:31,662 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_created():271] file/dir created: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-metadata.json
+2024-09-01 08:08:31,662 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_created():271] file/dir created: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:31,662 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_created():271] file/dir created: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/requirements.txt
+2024-09-01 08:08:33,662 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:35,486 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:37,663 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:39,664 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:40,093 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:40,094 DEBUG   SenderThread:1314 [sender.py:send():391] send: telemetry
+2024-09-01 08:08:40,800 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:40,813 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:08:41,665 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:42,978 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:42,979 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:42,979 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:42,980 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:43,666 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:43,666 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_created():271] file/dir created: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:45,667 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:45,808 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-01 08:08:45,808 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: stop_status
+2024-09-01 08:08:45,879 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:45,954 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:45,955 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:45,955 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:45,956 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:46,667 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:47,667 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:48,830 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:48,831 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:48,832 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:48,832 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:49,668 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:49,669 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:50,809 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:08:50,894 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:51,669 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:52,333 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:52,334 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:52,334 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:52,334 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:52,335 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:52,670 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:53,670 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:55,120 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:55,121 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:55,122 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:55,122 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:55,671 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:55,672 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:56,810 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:08:57,672 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:08:58,009 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:08:58,010 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:08:58,011 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:08:58,012 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:08:58,673 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:08:59,674 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:00,808 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-01 08:09:00,809 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:09:00,810 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: stop_status
+2024-09-01 08:09:00,953 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:09:00,954 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:09:00,955 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:09:00,955 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:01,674 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:01,675 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:02,613 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:03,675 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/config.yaml
+2024-09-01 08:09:03,676 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:03,815 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:09:03,817 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:09:03,817 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:09:03,817 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:04,676 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:05,676 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:07,247 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:09:07,248 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-01 08:09:07,249 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:09:07,249 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:09:07,250 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:07,678 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:07,678 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:08,251 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:09,678 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:10,825 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:09:13,826 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:15,810 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-01 08:09:15,811 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: stop_status
+2024-09-01 08:09:18,980 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:20,837 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:09:24,838 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:29,702 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: check_version
+2024-09-01 08:09:29,703 DEBUG   SenderThread:1314 [sender.py:send():391] send: telemetry
+2024-09-01 08:09:29,703 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: check_version
+2024-09-01 08:09:29,803 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: server_info
+2024-09-01 08:09:29,803 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: server_info
+2024-09-01 08:09:29,870 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:29,871 DEBUG   SenderThread:1314 [sender.py:send():391] send: exit
+2024-09-01 08:09:29,872 INFO    SenderThread:1314 [sender.py:send_exit():598] handling exit code: 0
+2024-09-01 08:09:29,872 INFO    SenderThread:1314 [sender.py:send_exit():600] handling runtime: 59
+2024-09-01 08:09:29,874 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:29,875 INFO    SenderThread:1314 [sender.py:send_exit():606] send defer
+2024-09-01 08:09:29,875 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,875 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 0
+2024-09-01 08:09:29,875 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,875 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 0
+2024-09-01 08:09:29,876 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 1
+2024-09-01 08:09:29,876 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,876 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 1
+2024-09-01 08:09:29,876 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,876 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 1
+2024-09-01 08:09:29,876 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 2
+2024-09-01 08:09:29,876 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,877 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 2
+2024-09-01 08:09:29,877 INFO    HandlerThread:1314 [system_monitor.py:finish():203] Stopping system monitor
+2024-09-01 08:09:29,877 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined cpu monitor
+2024-09-01 08:09:29,878 DEBUG   SystemMonitor:1314 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-09-01 08:09:29,878 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined disk monitor
+2024-09-01 08:09:29,878 DEBUG   SystemMonitor:1314 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-09-01 08:09:29,879 DEBUG   SystemMonitor:1314 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-09-01 08:09:29,895 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined gpu monitor
+2024-09-01 08:09:29,895 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined memory monitor
+2024-09-01 08:09:29,896 INFO    HandlerThread:1314 [interfaces.py:finish():200] Joined network monitor
+2024-09-01 08:09:29,896 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,896 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 2
+2024-09-01 08:09:29,896 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 3
+2024-09-01 08:09:29,896 DEBUG   SenderThread:1314 [sender.py:send():391] send: stats
+2024-09-01 08:09:29,897 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,897 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 3
+2024-09-01 08:09:29,899 DEBUG   SenderThread:1314 [sender.py:send():391] send: history
+2024-09-01 08:09:29,899 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: summary_record
+2024-09-01 08:09:29,899 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:29,900 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,900 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 3
+2024-09-01 08:09:29,900 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 4
+2024-09-01 08:09:29,900 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,900 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 4
+2024-09-01 08:09:29,901 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,901 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 4
+2024-09-01 08:09:29,901 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 5
+2024-09-01 08:09:29,901 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,901 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 5
+2024-09-01 08:09:29,901 DEBUG   SenderThread:1314 [sender.py:send():391] send: summary
+2024-09-01 08:09:29,902 INFO    SenderThread:1314 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2024-09-01 08:09:29,902 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,902 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 5
+2024-09-01 08:09:29,902 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 6
+2024-09-01 08:09:29,902 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:29,903 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 6
+2024-09-01 08:09:29,903 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:29,903 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 6
+2024-09-01 08:09:29,908 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: status_report
+2024-09-01 08:09:30,006 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 7
+2024-09-01 08:09:30,006 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:30,006 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 7
+2024-09-01 08:09:30,006 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:30,007 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 7
+2024-09-01 08:09:30,687 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/config.yaml
+2024-09-01 08:09:30,687 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:30,872 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: poll_exit
+2024-09-01 08:09:31,156 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 8
+2024-09-01 08:09:31,157 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: poll_exit
+2024-09-01 08:09:31,157 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:31,157 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 8
+2024-09-01 08:09:31,158 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:31,158 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 8
+2024-09-01 08:09:31,158 INFO    SenderThread:1314 [job_builder.py:build():443] Attempting to build job artifact
+2024-09-01 08:09:31,159 INFO    SenderThread:1314 [job_builder.py:_get_source_type():572] is repo sourced job
+2024-09-01 08:09:31,159 WARNING SenderThread:1314 [job_builder.py:_log_if_verbose():274] No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job
+2024-09-01 08:09:31,159 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 9
+2024-09-01 08:09:31,159 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:31,159 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 9
+2024-09-01 08:09:31,159 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:31,160 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 9
+2024-09-01 08:09:31,160 INFO    SenderThread:1314 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-09-01 08:09:31,687 INFO    Thread-12 :1314 [dir_watcher.py:_on_file_modified():288] file/dir modified: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:31,688 INFO    SenderThread:1314 [dir_watcher.py:finish():388] scan: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files
+2024-09-01 08:09:31,688 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/config.yaml config.yaml
+2024-09-01 08:09:31,688 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-metadata.json wandb-metadata.json
+2024-09-01 08:09:31,688 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/requirements.txt requirements.txt
+2024-09-01 08:09:31,691 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log output.log
+2024-09-01 08:09:31,694 INFO    SenderThread:1314 [dir_watcher.py:finish():402] scan save: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json wandb-summary.json
+2024-09-01 08:09:31,696 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 10
+2024-09-01 08:09:31,696 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:31,698 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 10
+2024-09-01 08:09:31,698 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:31,698 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 10
+2024-09-01 08:09:31,698 INFO    SenderThread:1314 [file_pusher.py:finish():169] shutting down file pusher
+2024-09-01 08:09:31,872 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: poll_exit
+2024-09-01 08:09:31,873 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: poll_exit
+2024-09-01 08:09:31,928 INFO    wandb-upload_0:1314 [upload_job.py:push():130] Uploaded file content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/config.yaml
+2024-09-01 08:09:32,038 INFO    wandb-upload_1:1314 [upload_job.py:push():130] Uploaded file content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/requirements.txt
+2024-09-01 08:09:32,135 INFO    wandb-upload_3:1314 [upload_job.py:push():130] Uploaded file content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/wandb-summary.json
+2024-09-01 08:09:32,170 INFO    wandb-upload_2:1314 [upload_job.py:push():130] Uploaded file content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/files/output.log
+2024-09-01 08:09:32,370 INFO    Thread-11 (_thread_body):1314 [sender.py:transition_state():626] send defer: 11
+2024-09-01 08:09:32,370 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:32,370 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 11
+2024-09-01 08:09:32,371 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:32,371 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 11
+2024-09-01 08:09:32,371 INFO    SenderThread:1314 [file_pusher.py:join():175] waiting for file pusher
+2024-09-01 08:09:32,372 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 12
+2024-09-01 08:09:32,372 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:32,372 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 12
+2024-09-01 08:09:32,372 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:32,373 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 12
+2024-09-01 08:09:32,373 INFO    SenderThread:1314 [file_stream.py:finish():601] file stream finish called
+2024-09-01 08:09:32,612 INFO    SenderThread:1314 [file_stream.py:finish():605] file stream finish is done
+2024-09-01 08:09:32,612 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 13
+2024-09-01 08:09:32,612 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:32,612 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 13
+2024-09-01 08:09:32,613 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:32,613 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 13
+2024-09-01 08:09:32,613 INFO    SenderThread:1314 [sender.py:transition_state():626] send defer: 14
+2024-09-01 08:09:32,613 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: defer
+2024-09-01 08:09:32,613 INFO    HandlerThread:1314 [handler.py:handle_request_defer():184] handle defer: 14
+2024-09-01 08:09:32,613 DEBUG   SenderThread:1314 [sender.py:send():391] send: final
+2024-09-01 08:09:32,614 DEBUG   SenderThread:1314 [sender.py:send():391] send: footer
+2024-09-01 08:09:32,614 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: defer
+2024-09-01 08:09:32,614 INFO    SenderThread:1314 [sender.py:send_request_defer():622] handle sender defer: 14
+2024-09-01 08:09:32,615 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: poll_exit
+2024-09-01 08:09:32,615 DEBUG   SenderThread:1314 [sender.py:send_request():418] send_request: poll_exit
+2024-09-01 08:09:32,616 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-01 08:09:32,616 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: get_summary
+2024-09-01 08:09:32,617 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: sampled_history
+2024-09-01 08:09:32,618 DEBUG   HandlerThread:1314 [handler.py:handle_request():158] handle_request: shutdown
+2024-09-01 08:09:32,618 INFO    HandlerThread:1314 [handler.py:finish():884] shutting down handler
+2024-09-01 08:09:33,615 INFO    WriterThread:1314 [datastore.py:close():296] close: content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/run-ra4ttuu9.wandb
+2024-09-01 08:09:33,615 INFO    SenderThread:1314 [sender.py:finish():1631] shutting down sender
+2024-09-01 08:09:33,616 INFO    SenderThread:1314 [file_pusher.py:finish():169] shutting down file pusher
+2024-09-01 08:09:33,616 INFO    SenderThread:1314 [file_pusher.py:join():175] waiting for file pusher

wandb/run-20240901_080830-ra4ttuu9/logs/debug.log

@@ -0,0 +1,36 @@
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Current SDK version is 0.17.8
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Configure stats pid to 1234
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Loading settings from /root/.config/wandb/settings
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Loading settings from /workspace/mistral-finetune/wandb/settings
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Loading settings from environment variables: {}
+2024-09-01 08:08:30,428 WARNING MainThread:1234 [wandb_setup.py:_flush():77] Could not find program at -m train
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m train'}
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Applying login settings: {'api_key': '***REDACTED***'}
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Applying login settings: {'api_key': '***REDACTED***'}
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_setup.py:_flush():77] Applying login settings: {}
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_init.py:_log_setup():524] Logging user logs to content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/logs/debug.log
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_init.py:_log_setup():525] Logging internal logs to content/test_ultra_1/wandb/run-20240901_080830-ra4ttuu9/logs/debug-internal.log
+2024-09-01 08:08:30,428 INFO    MainThread:1234 [wandb_init.py:init():607] calling init triggers
+2024-09-01 08:08:30,429 INFO    MainThread:1234 [wandb_init.py:init():614] wandb.init called with sweep_config: {}
+config: {'data': {'data': '', 'shuffle': False, 'instruct_data': './data/t_train.jsonl', 'eval_instruct_data': './data/t_eval.jsonl', 'instruct': {'shuffle': True, 'dynamic_chunk_fn_call': True}}, 'model_id_or_path': '../downloaded_model/mistral_models/7B-v0.3/', 'run_dir': './content/test_ultra_1', 'optim': {'lr': 0.0001, 'weight_decay': 0.1, 'pct_start': 0.05}, 'seed': 0, 'num_microbatches': 4, 'seq_len': 1024, 'batch_size': 1, 'max_norm': 1.0, 'max_steps': 10, 'log_freq': 1, 'ckpt_freq': 10, 'save_adapters': False, 'no_ckpt': False, 'num_ckpt_keep': 3, 'eval_freq': 5, 'no_eval': False, 'checkpoint': True, 'world_size': 1, 'wandb': {'project': 'finetuning_mistral', 'offline': False, 'key': '9e930bac040e715c4be9de064318956067b45479', 'run_name': None}, 'mlflow': {'tracking_uri': None, 'experiment_name': None}, 'lora': {'enable': True, 'rank': 16, 'dropout': 0.05, 'scaling': 2.0}}
+2024-09-01 08:08:30,429 INFO    MainThread:1234 [wandb_init.py:init():657] starting backend
+2024-09-01 08:08:30,429 INFO    MainThread:1234 [wandb_init.py:init():661] setting up manager
+2024-09-01 08:08:30,431 INFO    MainThread:1234 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-09-01 08:08:30,432 INFO    MainThread:1234 [wandb_init.py:init():669] backend started and connected
+2024-09-01 08:08:30,436 INFO    MainThread:1234 [wandb_init.py:init():767] updated telemetry
+2024-09-01 08:08:30,441 INFO    MainThread:1234 [wandb_init.py:init():800] communicating run to backend with 90.0 second timeout
+2024-09-01 08:08:30,665 INFO    MainThread:1234 [wandb_init.py:init():851] starting run threads in backend
+2024-09-01 08:08:30,808 INFO    MainThread:1234 [wandb_run.py:_console_start():2463] atexit reg
+2024-09-01 08:08:30,808 INFO    MainThread:1234 [wandb_run.py:_redirect():2309] redirect: wrap_raw
+2024-09-01 08:08:30,808 INFO    MainThread:1234 [wandb_run.py:_redirect():2374] Wrapping output streams.
+2024-09-01 08:08:30,808 INFO    MainThread:1234 [wandb_run.py:_redirect():2399] Redirects installed.
+2024-09-01 08:08:30,810 INFO    MainThread:1234 [wandb_init.py:init():894] run started, returning control to user process
+2024-09-01 08:09:29,670 INFO    MainThread:1234 [wandb_run.py:_finish():2160] finishing run dwivedi-rishabh95-avizva/finetuning_mistral/ra4ttuu9
+2024-09-01 08:09:29,671 INFO    MainThread:1234 [wandb_run.py:_atexit_cleanup():2424] got exitcode: 0
+2024-09-01 08:09:29,671 INFO    MainThread:1234 [wandb_run.py:_restore():2406] restore
+2024-09-01 08:09:29,671 INFO    MainThread:1234 [wandb_run.py:_restore():2412] restore done
+2024-09-01 08:09:29,671 INFO    MainThread:1234 [wandb_run.py:_on_finish():2677] communicating current version
+2024-09-01 08:09:29,802 INFO    MainThread:1234 [wandb_run.py:_on_finish():2686] got version response 
+2024-09-01 08:09:33,620 INFO    MainThread:1234 [wandb_run.py:_footer_history_summary_info():4078] rendering history
+2024-09-01 08:09:33,620 INFO    MainThread:1234 [wandb_run.py:_footer_history_summary_info():4110] rendering summary
+2024-09-01 08:09:33,629 INFO    MainThread:1234 [wandb_run.py:_footer_sync_info():4037] logging synced files