diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/debug-internal.log" @@ -0,0 +1,5568 @@ +2024-05-13 20:57:08,370 INFO StreamThr :1264247 [internal.py:wandb_internal():85] W&B internal server running at pid: 1264247, started at: 2024-05-13 20:57:08.369904 +2024-05-13 20:57:08,372 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status +2024-05-13 20:57:08,375 INFO WriterThread:1264247 [datastore.py:open_for_write():87] open: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/run-e3sq5zz5.wandb +2024-05-13 20:57:08,376 DEBUG SenderThread:1264247 [sender.py:send():378] send: header +2024-05-13 20:57:08,376 DEBUG SenderThread:1264247 [sender.py:send():378] send: run +2024-05-13 20:57:08,722 INFO SenderThread:1264247 [dir_watcher.py:__init__():211] watching files in: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files +2024-05-13 20:57:08,722 INFO SenderThread:1264247 [sender.py:_start_run_threads():1123] run started: e3sq5zz5 with start time 1715626628.368482 +2024-05-13 20:57:08,727 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: check_version +2024-05-13 20:57:08,728 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: check_version +2024-05-13 20:57:08,789 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: run_start +2024-05-13 20:57:08,836 DEBUG HandlerThread:1264247 [system_info.py:__init__():26] System info init +2024-05-13 20:57:08,836 DEBUG HandlerThread:1264247 [system_info.py:__init__():41] System info init done +2024-05-13 20:57:08,836 INFO HandlerThread:1264247 [system_monitor.py:start():194] Starting system monitor +2024-05-13 20:57:08,836 INFO SystemMonitor:1264247 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-13 20:57:08,836 INFO HandlerThread:1264247 [system_monitor.py:probe():214] Collecting system info +2024-05-13 20:57:08,836 INFO SystemMonitor:1264247 [interfaces.py:start():188] Started cpu monitoring +2024-05-13 20:57:08,837 INFO SystemMonitor:1264247 [interfaces.py:start():188] Started disk monitoring +2024-05-13 20:57:08,838 INFO SystemMonitor:1264247 [interfaces.py:start():188] Started gpu monitoring +2024-05-13 20:57:08,839 INFO SystemMonitor:1264247 [interfaces.py:start():188] Started memory monitoring +2024-05-13 20:57:08,839 INFO SystemMonitor:1264247 [interfaces.py:start():188] Started network monitoring +2024-05-13 20:57:08,881 DEBUG HandlerThread:1264247 [system_info.py:probe():150] Probing system +2024-05-13 20:57:08,882 DEBUG HandlerThread:1264247 [system_info.py:_probe_git():135] Probing git +2024-05-13 20:57:08,887 DEBUG HandlerThread:1264247 [system_info.py:_probe_git():143] Probing git done +2024-05-13 20:57:08,887 DEBUG HandlerThread:1264247 [system_info.py:probe():198] Probing system done +2024-05-13 20:57:08,887 DEBUG HandlerThread:1264247 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.31', 'python': '3.11.9', 'heartbeatAt': '2024-05-13T18:57:08.881050', 'startedAt': '2024-05-13T18:57:08.362801', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path', 'parler-tts/parler_tts_mini_v0.1', '--feature_extractor_name', 'parler-tts/dac_44khZ_8kbps', '--description_tokenizer_name', 'parler-tts/parler_tts_mini_v0.1', '--prompt_tokenizer_name', 'parler-tts/parler_tts_mini_v0.1', '--report_to', 'wandb', '--overwrite_output_dir', 'true', '--train_dataset_name', 'sanchit-gandhi/expresso-concatenated-half-normal', '--train_metadata_dataset_name', 'sanchit-gandhi/expresso-concatenated-half-normal-tags-mistral', '--train_dataset_config_name', 'read', '--train_split_name', 'train[:-35]', '--eval_dataset_name', 'sanchit-gandhi/expresso-concatenated-half-normal', '--eval_metadata_dataset_name', 'sanchit-gandhi/expresso-concatenated-half-normal-tags-mistral', '--eval_dataset_config_name', 'read', '--eval_split_name', 'train', '--max_eval_samples', '8', '--per_device_eval_batch_size', '16', '--target_audio_column_name', 'audio', '--description_column_name', 'text_description', '--prompt_column_name', 'text', '--max_duration_in_seconds', '30.0', '--min_duration_in_seconds', '2.0', '--max_text_length', '400', '--preprocessing_num_workers', '2', '--do_train', 'true', '--num_train_epochs', '8', '--gradient_accumulation_steps', '8', '--gradient_checkpointing', 'true', '--per_device_train_batch_size', '16', '--learning_rate', '0.00008', '--adam_beta1', '0.9', '--adam_beta2', '0.99', '--weight_decay', '0.01', '--lr_scheduler_type', 'cosine', '--warmup_steps', '250', '--logging_steps', '2', '--freeze_text_encoder', 'true', '--audio_encoder_per_device_batch_size', '4', '--dtype', 'bfloat16', '--seed', '456', '--output_dir', './', '--temporary_save_to_disk', '../audio_code_tmp_concat/', '--save_to_disk', '../tmp_dataset_audio_concat/', '--dataloader_num_workers', '4', '--do_eval', '--predict_with_generate', '--include_inputs_for_metrics', '--save_strategy', 'steps', '--save_steps', '72', '--evaluation_strategy', 'epoch', '--save_total_limit', '5', '--group_by_length', 'true'), 'state': 'running', 'program': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/./run_parler_tts_training.py', 'codePathLocal': 'run_parler_tts_training.py', 'codePath': 'run_parler_tts_training.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/parler-tts-mini-v0.1-expresso-concatenated', 'commit': '99f75adb5e13ee0ad87cce8deb3e71adc370b10e'}, 'email': 'sanchit@huggingface.co', 'root': '/raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/miniconda3/envs/venv/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2704.8252031250004, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 2268.347, 'min': 1500.0, 'max': 2250.0}, {'current': 3312.302, 'min': 1500.0, 'max': 2250.0}, {'current': 1715.829, 'min': 1500.0, 'max': 2250.0}, {'current': 1716.824, 'min': 1500.0, 'max': 2250.0}, {'current': 2341.495, 'min': 1500.0, 'max': 2250.0}, {'current': 1712.234, 'min': 1500.0, 'max': 2250.0}, {'current': 1717.216, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.034, 'min': 1500.0, 'max': 2250.0}, {'current': 3336.893, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.378, 'min': 1500.0, 'max': 2250.0}, {'current': 2995.971, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.893, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.345, 'min': 1500.0, 'max': 2250.0}, {'current': 2374.632, 'min': 1500.0, 'max': 2250.0}, {'current': 3305.389, 'min': 1500.0, 'max': 2250.0}, {'current': 2686.42, 'min': 1500.0, 'max': 2250.0}, {'current': 3148.45, 'min': 1500.0, 'max': 2250.0}, {'current': 1674.683, 'min': 1500.0, 'max': 2250.0}, {'current': 1674.39, 'min': 1500.0, 'max': 2250.0}, {'current': 1717.121, 'min': 1500.0, 'max': 2250.0}, {'current': 2590.333, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.301, 'min': 1500.0, 'max': 2250.0}, {'current': 1715.006, 'min': 1500.0, 'max': 2250.0}, {'current': 1958.198, 'min': 1500.0, 'max': 2250.0}, {'current': 1717.663, 'min': 1500.0, 'max': 2250.0}, {'current': 1712.57, 'min': 1500.0, 'max': 2250.0}, {'current': 2460.892, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.872, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.436, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.171, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.817, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.368, 'min': 1500.0, 'max': 2250.0}, {'current': 3298.132, 'min': 1500.0, 'max': 2250.0}, {'current': 2587.095, 'min': 1500.0, 'max': 2250.0}, {'current': 2345.42, 'min': 1500.0, 'max': 2250.0}, {'current': 1660.028, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.857, 'min': 1500.0, 'max': 2250.0}, {'current': 1667.983, 'min': 1500.0, 'max': 2250.0}, {'current': 1847.396, 'min': 1500.0, 'max': 2250.0}, {'current': 2553.294, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.979, 'min': 1500.0, 'max': 2250.0}, {'current': 3174.24, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.394, 'min': 1500.0, 'max': 2250.0}, {'current': 1712.395, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.273, 'min': 1500.0, 'max': 2250.0}, {'current': 2318.965, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.578, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.36, 'min': 1500.0, 'max': 2250.0}, {'current': 2185.7, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.83, 'min': 1500.0, 'max': 2250.0}, {'current': 3309.021, 'min': 1500.0, 'max': 2250.0}, {'current': 3326.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3341.639, 'min': 1500.0, 'max': 2250.0}, {'current': 3308.608, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.855, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.45, 'min': 1500.0, 'max': 2250.0}, {'current': 1658.293, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.748, 'min': 1500.0, 'max': 2250.0}, {'current': 3325.253, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.206, 'min': 1500.0, 'max': 2250.0}, {'current': 3279.351, 'min': 1500.0, 'max': 2250.0}, {'current': 3037.252, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.356, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.844, 'min': 1500.0, 'max': 2250.0}, {'current': 2142.747, 'min': 1500.0, 'max': 2250.0}, {'current': 3314.624, 'min': 1500.0, 'max': 2250.0}, {'current': 2128.073, 'min': 1500.0, 'max': 2250.0}, {'current': 2150.803, 'min': 1500.0, 'max': 2250.0}, {'current': 2146.302, 'min': 1500.0, 'max': 2250.0}, {'current': 2171.861, 'min': 1500.0, 'max': 2250.0}, {'current': 2159.384, 'min': 1500.0, 'max': 2250.0}, {'current': 2174.728, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.354, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.023, 'min': 1500.0, 'max': 2250.0}, {'current': 2285.509, 'min': 1500.0, 'max': 2250.0}, {'current': 1784.023, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.135, 'min': 1500.0, 'max': 2250.0}, {'current': 3138.985, 'min': 1500.0, 'max': 2250.0}, {'current': 3298.618, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.483, 'min': 1500.0, 'max': 2250.0}, {'current': 3307.703, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.991, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.301, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.895, 'min': 1500.0, 'max': 2250.0}, {'current': 3318.153, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.383, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.4, 'min': 1500.0, 'max': 2250.0}, {'current': 3306.436, 'min': 1500.0, 'max': 2250.0}, {'current': 1658.878, 'min': 1500.0, 'max': 2250.0}, {'current': 1654.849, 'min': 1500.0, 'max': 2250.0}, {'current': 3300.978, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.09, 'min': 1500.0, 'max': 2250.0}, {'current': 1736.603, 'min': 1500.0, 'max': 2250.0}, {'current': 1735.507, 'min': 1500.0, 'max': 2250.0}, {'current': 1733.054, 'min': 1500.0, 'max': 2250.0}, {'current': 1780.249, 'min': 1500.0, 'max': 2250.0}, {'current': 3318.8, 'min': 1500.0, 'max': 2250.0}, {'current': 3308.615, 'min': 1500.0, 'max': 2250.0}, {'current': 3329.812, 'min': 1500.0, 'max': 2250.0}, {'current': 1654.782, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.907, 'min': 1500.0, 'max': 2250.0}, {'current': 1668.539, 'min': 1500.0, 'max': 2250.0}, {'current': 2168.481, 'min': 1500.0, 'max': 2250.0}, {'current': 3319.301, 'min': 1500.0, 'max': 2250.0}, {'current': 2356.902, 'min': 1500.0, 'max': 2250.0}, {'current': 3298.032, 'min': 1500.0, 'max': 2250.0}, {'current': 1740.676, 'min': 1500.0, 'max': 2250.0}, {'current': 2390.405, 'min': 1500.0, 'max': 2250.0}, {'current': 2375.499, 'min': 1500.0, 'max': 2250.0}, {'current': 3296.073, 'min': 1500.0, 'max': 2250.0}, {'current': 2284.626, 'min': 1500.0, 'max': 2250.0}, {'current': 1702.39, 'min': 1500.0, 'max': 2250.0}, {'current': 3298.714, 'min': 1500.0, 'max': 2250.0}, {'current': 2383.361, 'min': 1500.0, 'max': 2250.0}, {'current': 3287.088, 'min': 1500.0, 'max': 2250.0}, {'current': 3298.321, 'min': 1500.0, 'max': 2250.0}, {'current': 3298.689, 'min': 1500.0, 'max': 2250.0}, {'current': 3306.411, 'min': 1500.0, 'max': 2250.0}, {'current': 2384.963, 'min': 1500.0, 'max': 2250.0}, {'current': 2392.568, 'min': 1500.0, 'max': 2250.0}, {'current': 1658.572, 'min': 1500.0, 'max': 2250.0}, {'current': 2361.045, 'min': 1500.0, 'max': 2250.0}, {'current': 3298.714, 'min': 1500.0, 'max': 2250.0}, {'current': 1652.106, 'min': 1500.0, 'max': 2250.0}, {'current': 3305.482, 'min': 1500.0, 'max': 2250.0}, {'current': 1724.1, 'min': 1500.0, 'max': 2250.0}, {'current': 1650.133, 'min': 1500.0, 'max': 2250.0}, {'current': 1652.413, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1663.505687713623}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-05-13 20:57:08,888 INFO HandlerThread:1264247 [system_monitor.py:probe():224] Finished collecting system info +2024-05-13 20:57:08,888 INFO HandlerThread:1264247 [system_monitor.py:probe():227] Publishing system info +2024-05-13 20:57:08,888 DEBUG HandlerThread:1264247 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-13 20:57:08,905 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:08,921 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:09,723 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/conda-environment.yaml +2024-05-13 20:57:10,946 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:10,956 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:11,206 DEBUG HandlerThread:1264247 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-13 20:57:11,207 INFO HandlerThread:1264247 [system_monitor.py:probe():229] Finished publishing system info +2024-05-13 20:57:11,215 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 20:57:11,216 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-metadata.json with policy now +2024-05-13 20:57:11,329 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: python_packages +2024-05-13 20:57:11,329 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:11,330 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: python_packages +2024-05-13 20:57:11,331 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:11,525 DEBUG SenderThread:1264247 [sender.py:send():378] send: telemetry +2024-05-13 20:57:11,525 DEBUG SenderThread:1264247 [sender.py:send():378] send: config +2024-05-13 20:57:11,723 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/conda-environment.yaml +2024-05-13 20:57:11,723 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/requirements.txt +2024-05-13 20:57:11,723 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:11,723 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-metadata.json +2024-05-13 20:57:11,729 INFO wandb-upload_0:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/r9h9iwty-wandb-metadata.json +2024-05-13 20:57:12,981 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:12,992 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:13,649 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:13,723 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:15,724 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:16,178 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:16,189 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:17,724 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:18,214 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:18,232 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:19,274 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:19,725 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:20,261 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:20,276 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:21,725 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:22,298 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:22,310 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:23,726 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:24,322 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:25,563 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:25,574 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:26,328 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:26,329 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:27,597 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:27,611 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:29,421 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:29,633 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:29,648 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:29,728 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:31,682 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:31,705 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:31,728 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:34,577 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:35,047 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:35,061 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:37,141 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:37,152 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:37,730 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:39,181 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:39,192 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:39,730 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:40,139 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:40,730 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/config.yaml +2024-05-13 20:57:41,217 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:41,231 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:41,328 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:41,329 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:45,139 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:57:45,161 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:45,171 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:45,692 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:45,732 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:48,390 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:48,418 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:50,453 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:50,475 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:50,694 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:51,734 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:52,496 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:52,508 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:53,734 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:55,733 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:55,762 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:56,304 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:57:56,328 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:57:56,329 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:57:57,817 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:57,830 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:59,736 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:57:59,852 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:57:59,863 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:01,511 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:01,736 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:01,888 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:01,901 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:03,738 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:03,930 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:03,945 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:05,983 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:05,995 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:06,741 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:07,739 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:08,840 DEBUG SystemMonitor:1264247 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-13 20:58:08,891 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 20:58:09,301 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:09,312 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:09,741 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:11,328 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:11,329 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:11,418 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:11,430 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:11,741 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:11,837 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:13,578 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:13,590 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:13,742 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:15,742 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:15,788 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:15,809 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:17,027 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:17,743 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:17,951 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:17,989 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:19,744 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:20,716 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:58:20,749 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:20,762 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:21,745 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:22,144 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:22,785 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:22,805 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:23,746 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:24,832 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:24,845 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:25,746 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:26,329 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:26,329 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:26,866 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:26,877 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:27,200 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:27,747 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:28,906 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:28,923 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:29,748 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:31,062 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:31,090 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:31,748 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:32,278 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:33,233 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:33,263 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:33,750 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:35,401 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:35,437 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:35,750 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:37,489 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:37,514 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:37,550 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:37,751 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:38,893 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 20:58:39,751 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:39,752 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:39,788 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:41,329 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:41,329 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:41,752 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:41,994 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:42,024 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:42,606 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:43,753 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:44,185 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:44,211 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:45,754 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:46,383 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:46,400 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:47,683 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:47,754 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:48,546 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:48,568 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:49,755 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:50,724 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:50,734 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:51,755 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:52,723 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:53,527 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:58:53,559 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:53,756 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:54,290 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:55,757 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:56,329 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:58:56,330 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:58:57,329 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:57,352 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:57,735 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:58:57,758 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:58:59,502 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:59,534 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:58:59,758 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:01,699 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:01,742 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:01,759 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:02,786 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:03,760 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:03,901 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:03,937 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:05,760 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:06,064 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:06,093 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:07,761 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:07,915 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:08,228 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:08,261 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:08,895 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 20:59:09,762 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:10,409 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:10,430 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:11,329 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:11,329 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:11,762 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:12,595 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:12,624 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:12,956 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:13,763 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:14,772 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:14,796 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:15,764 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:16,996 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:17,026 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:17,764 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:18,195 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:19,170 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:19,207 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:19,765 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:21,343 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:21,373 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:21,766 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:23,232 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:23,503 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:23,515 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:23,766 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:25,705 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:25,731 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:25,767 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:26,329 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:26,330 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:27,768 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:28,300 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:28,475 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 20:59:28,506 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:28,517 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:29,768 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:30,538 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:30,550 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:31,769 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:32,593 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:32,604 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:33,420 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:33,770 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:34,629 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:34,643 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:35,770 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:36,732 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:36,746 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:37,771 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:38,463 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:38,888 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:38,897 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 20:59:38,917 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:39,771 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:41,033 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:41,068 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:41,376 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:41,377 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:41,773 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:43,202 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:43,237 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:43,535 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:43,773 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:45,410 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:45,451 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:45,773 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:47,612 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:47,654 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:47,773 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:48,632 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:49,774 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:49,785 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:49,808 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:51,774 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:51,949 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:51,982 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:53,698 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:53,775 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:54,130 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:54,163 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:55,776 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:56,361 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:56,376 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 20:59:56,377 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 20:59:56,394 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:57,776 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 20:59:58,516 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:58,530 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 20:59:58,820 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 20:59:59,777 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:01,293 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:00:01,328 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:01,350 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:01,778 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:03,376 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:03,394 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:03,778 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:03,902 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:05,418 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:05,432 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:05,779 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:07,454 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:07,466 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:07,780 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:08,898 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:00:09,017 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:09,495 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:09,517 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:09,780 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:11,376 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:11,377 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:11,538 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:11,548 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:11,781 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:13,571 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:13,583 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:13,782 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:14,047 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:15,730 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:15,767 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:15,782 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:17,783 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:17,798 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:17,830 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:19,191 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:19,783 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:19,879 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:19,913 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:21,784 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:22,032 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:22,065 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:23,785 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:24,232 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:24,273 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:24,300 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:25,785 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:26,376 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:26,377 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:26,468 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:26,499 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:27,786 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:28,647 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:28,684 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:29,418 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:29,786 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:30,887 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:30,914 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:31,787 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:33,787 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:34,612 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:34,713 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:00:35,194 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:35,216 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:35,788 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:37,357 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:37,390 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:37,789 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:38,902 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:00:39,551 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:39,580 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:39,714 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:39,789 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:41,376 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:41,377 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:41,705 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:41,731 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:41,790 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:43,790 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:43,919 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:43,952 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:44,715 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:45,791 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:46,121 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:46,150 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:47,791 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:48,240 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:48,265 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:49,792 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:49,839 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:50,302 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:50,330 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:51,792 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:52,353 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:52,374 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:54,411 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:54,432 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:54,890 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:00:56,393 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:00:56,394 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:00:56,454 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:56,469 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:57,794 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:58,513 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:58,524 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:00:59,795 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:00:59,943 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:01,729 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:01,770 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:03,863 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:03,921 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:05,006 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:05,796 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:05,964 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:05,987 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:07,797 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:08,903 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:01:09,798 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:09,982 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:01:10,006 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:10,028 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:10,028 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:11,392 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:11,393 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:11,799 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:12,067 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:12,097 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:13,800 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:14,124 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:14,138 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:15,455 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:15,801 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:17,320 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:17,340 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:17,801 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:20,133 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:20,147 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:20,863 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:22,252 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:22,286 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:23,803 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:24,319 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:24,350 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:25,804 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:26,057 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:26,392 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:26,393 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:27,570 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:27,599 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:29,689 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:29,748 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:31,253 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:31,775 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:31,790 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:31,805 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:33,806 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:33,819 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:33,833 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:35,861 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:35,873 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:36,565 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:37,937 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:37,962 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:38,905 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:01:39,808 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:40,012 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:40,024 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:41,392 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:41,393 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:41,624 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:41,808 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:42,062 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:42,084 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:45,810 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:46,051 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:01:46,076 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:46,090 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:46,688 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:47,810 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:48,131 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:48,149 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:51,289 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:51,322 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:52,686 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:53,343 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:53,365 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:53,812 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:55,386 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:55,397 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:55,812 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:56,393 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:01:56,393 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:01:57,813 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:01:58,285 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:01:58,358 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:58,383 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:01:59,814 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:02:00,422 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:00,445 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:02,565 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:02,585 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:03,474 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:04,758 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:04,799 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:07,003 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:07,029 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:08,475 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:08,908 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:02:09,223 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:09,235 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:11,343 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:11,387 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:11,394 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:11,395 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:13,496 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:13,528 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:13,586 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:15,578 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:15,605 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:17,668 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:17,698 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:18,587 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:19,819 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:02:21,589 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:02:21,620 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:21,645 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:23,674 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:24,066 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:24,917 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:26,393 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:26,394 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:26,950 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:26,974 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:27,663 DEBUG SenderThread:1264247 [sender.py:send():378] send: telemetry +2024-05-13 21:02:27,664 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:02:28,997 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:29,020 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:29,664 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:29,822 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:02:31,041 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:31,055 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:33,105 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:33,148 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:33,823 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:02:34,733 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:36,157 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:36,185 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:36,650 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:02:36,651 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:02:36,651 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:02:36,651 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:02:36,823 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:02:37,824 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:02:38,911 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:02:39,385 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:39,408 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:39,824 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:02:40,329 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:41,381 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:02:41,382 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:02:41,383 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:02:41,383 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:02:41,393 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:41,393 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:41,470 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:41,528 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:41,825 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:02:41,825 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:02:43,572 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:43,587 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:43,825 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:02:45,618 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:46,749 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:46,783 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:48,807 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:48,845 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:50,623 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:50,827 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/config.yaml +2024-05-13 21:02:50,865 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:50,875 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:52,896 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:52,907 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:54,937 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:54,991 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:02:55,789 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:02:56,398 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:02:56,399 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:02:59,833 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:03:00,478 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:03:00,564 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:00,614 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:01,453 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:02,650 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:02,709 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:04,794 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:04,817 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:06,454 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:06,862 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:06,884 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:08,754 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:03:08,755 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:03:08,755 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:03:08,755 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:03:08,835 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:03:08,914 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:03:08,929 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:08,954 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:09,836 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:03:10,992 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:11,030 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:11,399 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:11,399 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:11,565 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:13,061 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:13,089 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:15,154 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:15,202 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:15,837 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:03:16,666 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:18,479 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:18,514 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:18,884 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:03:18,885 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:03:18,885 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:03:18,885 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:03:19,838 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:03:19,838 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:03:20,587 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:20,615 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:21,886 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:22,695 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:22,738 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:23,839 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:03:24,366 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:03:24,367 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:03:24,367 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:03:24,367 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:03:24,807 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:24,839 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:03:24,839 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:25,839 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:03:26,399 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:26,399 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:27,349 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:27,375 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:27,587 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:29,407 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:29,429 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:31,459 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:31,473 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:32,588 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:35,398 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:03:35,420 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:35,441 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:37,588 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:38,762 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:38,797 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:38,916 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:03:40,819 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:40,838 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:41,399 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:41,399 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:41,844 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:03:42,856 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:42,867 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:43,584 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:44,891 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:44,902 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:48,241 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:48,264 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:48,585 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:50,346 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:50,389 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:51,804 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:03:51,805 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:03:51,805 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:03:51,806 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:03:51,846 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:03:52,424 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:52,439 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:53,807 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:53,847 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:03:55,569 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:55,592 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:56,399 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:03:56,399 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:03:57,619 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:57,638 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:59,486 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:03:59,662 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:59,681 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:03:59,849 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:01,700 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:01,712 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:03,006 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:04:03,006 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:04:03,007 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:04:03,454 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:04:03,739 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:03,748 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:03,850 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:04:03,850 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:05,455 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:05,767 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:05,780 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:07,798 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:07,809 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:07,851 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:08,918 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:04:09,194 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:04:09,195 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:04:09,196 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:04:09,196 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:04:09,852 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:04:09,852 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:11,196 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:11,399 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:11,399 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:11,852 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:13,207 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:04:13,258 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:13,271 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:15,383 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:15,404 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:16,637 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:17,478 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:17,498 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:19,578 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:19,602 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:21,706 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:21,729 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:21,855 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:22,076 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:23,814 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:23,834 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:25,929 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:25,955 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:26,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:26,400 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:27,569 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:28,054 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:28,094 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:30,203 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:30,240 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:32,340 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:32,377 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:32,570 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:34,487 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:34,525 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:36,477 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:04:36,478 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:04:36,478 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:04:36,479 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:04:36,631 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:36,663 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:36,859 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:04:37,859 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:38,479 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:38,802 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:38,827 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:38,921 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:04:40,963 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:40,994 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:41,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:41,400 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:43,107 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:43,122 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:43,542 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:45,861 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:47,200 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:04:47,228 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:47,253 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:49,281 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:49,298 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:49,485 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:50,311 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:04:50,312 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:04:50,312 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:04:50,313 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:04:50,863 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:04:51,321 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:51,336 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:51,863 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:53,360 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:53,370 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:54,818 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:04:55,397 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:55,413 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:55,865 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:56,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:04:56,400 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:04:56,405 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:04:56,628 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:04:56,628 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:04:56,629 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:04:56,865 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:04:57,452 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:57,464 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:57,865 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:04:59,499 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:59,509 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:04:59,866 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:05:00,244 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:01,532 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:01,547 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:03,574 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:03,588 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:05,245 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:05,619 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:05,636 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:07,676 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:07,692 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:08,923 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:05:09,739 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:09,754 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:10,924 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:11,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:11,400 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:11,861 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:11,882 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:13,997 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:14,024 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:16,076 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:16,097 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:16,606 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:19,010 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:05:19,011 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:05:19,012 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:05:19,012 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:05:19,874 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:05:19,874 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:05:20,125 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:05:20,172 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:20,203 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:22,013 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:22,229 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:22,256 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:24,292 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:24,324 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:26,378 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:26,396 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:26,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:26,400 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:27,322 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:27,877 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:05:28,440 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:28,465 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:30,487 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:30,520 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:32,325 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:32,549 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:32,572 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:33,801 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:05:33,802 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:05:33,802 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:05:33,803 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:05:33,879 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:05:34,599 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:34,614 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:35,880 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:05:36,665 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:36,689 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:37,472 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:37,881 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:05:38,754 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:38,793 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:38,925 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:05:40,289 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:05:40,290 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:05:40,290 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:05:40,290 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:05:40,882 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:05:40,894 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:40,919 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:41,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:41,401 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:41,882 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:05:42,981 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:43,019 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:43,282 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:43,883 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:05:45,117 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:45,153 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:47,276 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:47,305 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:48,283 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:49,400 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:49,424 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:53,283 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:53,491 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:05:53,543 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:53,570 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:55,631 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:55,666 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:56,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:05:56,401 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:05:57,792 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:57,820 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:58,629 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:05:59,732 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:05:59,733 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:05:59,733 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:05:59,734 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:05:59,888 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:05:59,919 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:05:59,955 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:01,888 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:02,095 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:02,120 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:03,735 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:04,243 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:04,288 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:06,416 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:06,457 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:08,520 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:08,559 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:08,735 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:08,928 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:06:10,638 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:10,675 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:11,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:11,401 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:11,891 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:12,756 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:12,785 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:14,557 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:14,972 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:15,004 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:16,158 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:06:16,159 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:06:16,159 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:06:16,160 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:06:16,893 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:06:17,093 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:17,122 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:17,893 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:19,196 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:19,236 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:20,161 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:21,275 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:21,298 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:21,895 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:23,337 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:23,365 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:23,467 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:06:23,468 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:06:23,468 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:06:23,468 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:06:23,895 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:06:23,895 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:25,469 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:25,896 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:26,401 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:26,401 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:27,452 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:06:27,476 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:27,488 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:27,897 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:29,514 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:29,528 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:30,592 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:31,550 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:31,562 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:33,583 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:33,596 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:35,593 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:35,677 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:35,689 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:37,325 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:06:37,326 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:06:37,326 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:06:37,327 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:06:37,851 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:37,895 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:37,899 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:06:37,899 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:38,929 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:06:39,900 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:39,966 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:39,992 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:40,931 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:41,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:41,401 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:42,065 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:42,111 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:43,901 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:06:44,160 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:44,184 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:46,211 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:46,229 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:46,649 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:48,254 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:48,266 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:50,287 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:50,302 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:51,650 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:53,261 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:53,583 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:55,604 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:55,618 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:56,401 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:06:56,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:06:57,605 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:06:57,647 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:06:57,665 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:01,593 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:07:01,615 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:01,633 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:02,606 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:03,662 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:03,682 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:05,710 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:05,741 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:07,607 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:07,761 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:07,780 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:08,932 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:07:09,812 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:09,835 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:11,401 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:11,401 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:11,862 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:11,878 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:13,601 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:15,200 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:15,224 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:17,269 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:17,290 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:18,602 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:19,325 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:19,350 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:21,371 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:21,398 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:21,910 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:07:23,910 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:07:24,045 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:24,567 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:24,590 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:24,605 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 21:07:24,605 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 21:07:24,911 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/table/predictions/eval-step-39_38_a379d6cee0305d384048.table.json +2024-05-13 21:07:24,911 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/table +2024-05-13 21:07:24,911 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/table/predictions +2024-05-13 21:07:24,912 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media +2024-05-13 21:07:24,915 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:07:24,971 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:07:24,972 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:07:25,346 INFO wandb-upload_0:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/v5flzow8-media/table/predictions/eval-step-39_38_a379d6cee0305d384048.table.json +2024-05-13 21:07:25,731 INFO wandb-upload_1:1264247 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmprwykc9wx +2024-05-13 21:07:25,911 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_39_75f77b4060ad41f6cd28.wav +2024-05-13 21:07:25,912 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_39_a92c59803d84d120f9c8.wav +2024-05-13 21:07:25,912 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_39_e93fbf8d0314875d9a8a.wav +2024-05-13 21:07:25,912 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_39_91ec273933f07274531b.wav +2024-05-13 21:07:25,912 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_39_6481b4598446a5223728.wav +2024-05-13 21:07:25,912 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_39_e334e92b9d2007210460.wav +2024-05-13 21:07:25,913 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_39_65c5863263ecc0063e7c.wav +2024-05-13 21:07:25,913 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media +2024-05-13 21:07:25,913 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio +2024-05-13 21:07:25,913 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples +2024-05-13 21:07:26,365 INFO SenderThread:1264247 [sender.py:send_request_log_artifact():1455] logged artifact run-e3sq5zz5-predictionsevalstep39 - {'id': 'QXJ0aWZhY3Q6ODM0NzUzNDA5', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI0MTI2Mw==', 'latestArtifact': None}} +2024-05-13 21:07:26,365 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:07:26,365 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-39_38_a379d6cee0305d384048.table.json with policy now +2024-05-13 21:07:26,366 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:07:26,366 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:07:26,369 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:07:26,369 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:07:26,369 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_39_65c5863263ecc0063e7c.wav with policy now +2024-05-13 21:07:26,370 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:07:26,371 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_39_75f77b4060ad41f6cd28.wav with policy now +2024-05-13 21:07:26,373 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:07:26,373 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_39_a92c59803d84d120f9c8.wav with policy now +2024-05-13 21:07:26,373 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:07:26,373 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_39_e334e92b9d2007210460.wav with policy now +2024-05-13 21:07:26,374 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:07:26,374 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_39_e93fbf8d0314875d9a8a.wav with policy now +2024-05-13 21:07:26,375 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:07:26,375 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_39_6481b4598446a5223728.wav with policy now +2024-05-13 21:07:26,376 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:07:26,376 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_39_91ec273933f07274531b.wav with policy now +2024-05-13 21:07:26,401 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:26,401 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:26,650 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:26,662 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:26,912 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:07:26,989 INFO wandb-upload_7:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/2dk1rwxb-media/table/predictions/eval-step-39_38_a379d6cee0305d384048.table.json +2024-05-13 21:07:27,389 INFO wandb-upload_0:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/yozclo7p-media/audio/Speech samples/eval_39_a92c59803d84d120f9c8.wav +2024-05-13 21:07:27,468 INFO wandb-upload_2:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/rg61n7il-media/audio/Speech samples/eval_39_75f77b4060ad41f6cd28.wav +2024-05-13 21:07:27,509 INFO wandb-upload_3:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/thvsu1os-media/audio/Speech samples/eval_39_91ec273933f07274531b.wav +2024-05-13 21:07:27,517 INFO wandb-upload_4:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/3cbatf5g-media/audio/Speech samples/eval_39_6481b4598446a5223728.wav +2024-05-13 21:07:27,554 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:07:27,556 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:07:27,556 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:07:27,556 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:07:27,570 INFO wandb-upload_5:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/omxutjci-media/audio/Speech samples/eval_39_e334e92b9d2007210460.wav +2024-05-13 21:07:27,579 INFO wandb-upload_1:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/jjvepfsz-media/audio/Speech samples/eval_39_e93fbf8d0314875d9a8a.wav +2024-05-13 21:07:27,590 INFO wandb-upload_6:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/80ixhbqx-media/audio/Speech samples/eval_39_65c5863263ecc0063e7c.wav +2024-05-13 21:07:27,912 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:07:27,913 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:07:28,520 INFO wandb-upload_7:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/b7g2sh39-media/audio/Speech samples/eval_39_a92c59803d84d120f9c8.wav +2024-05-13 21:07:28,530 INFO wandb-upload_3:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/1wv46zg9-media/audio/Speech samples/eval_39_91ec273933f07274531b.wav +2024-05-13 21:07:28,616 INFO wandb-upload_2:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/54zswtw8-media/audio/Speech samples/eval_39_6481b4598446a5223728.wav +2024-05-13 21:07:28,664 INFO wandb-upload_5:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/252rosn0-media/audio/Speech samples/eval_39_65c5863263ecc0063e7c.wav +2024-05-13 21:07:28,707 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:28,720 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:29,498 INFO wandb-upload_6:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/of7rojhb-media/audio/Speech samples/eval_39_e334e92b9d2007210460.wav +2024-05-13 21:07:29,557 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:29,686 INFO wandb-upload_1:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/ur6dtf75-media/audio/Speech samples/eval_39_75f77b4060ad41f6cd28.wav +2024-05-13 21:07:29,734 INFO wandb-upload_7:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/f8r7zwpj-media/audio/Speech samples/eval_39_e93fbf8d0314875d9a8a.wav +2024-05-13 21:07:29,913 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:07:30,775 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:30,787 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:32,832 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:32,846 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:34,558 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:36,977 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:07:37,003 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:37,018 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:38,935 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:07:39,049 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:39,064 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:39,938 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:41,087 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:41,098 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:41,401 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:41,401 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:43,125 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:43,144 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:45,173 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:45,192 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:45,570 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:47,214 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:47,225 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:49,250 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:49,261 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:49,923 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:07:51,285 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:51,304 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:51,339 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:53,366 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:53,376 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:55,449 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:55,463 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:56,340 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:07:56,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:07:56,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:07:57,548 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:57,582 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:58,774 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:07:58,776 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:07:58,776 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:07:58,776 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:07:58,926 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:07:59,704 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:59,731 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:07:59,927 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:01,777 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:01,827 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:01,870 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:03,976 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:04,000 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:05,929 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:06,117 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:06,146 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:06,941 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:07,741 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:08:07,743 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:08:07,743 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:08:07,743 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:08:07,930 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:08:08,937 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:08:09,931 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:10,271 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:08:10,326 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:10,359 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:11,401 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:11,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:11,932 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:12,399 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:12,439 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:12,471 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:12,515 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:08:12,516 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:08:12,516 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:08:12,517 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:08:12,933 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:08:13,933 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:14,491 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:14,523 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:16,624 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:16,653 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:17,520 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:18,699 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:18,717 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:20,751 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:20,775 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:22,521 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:22,795 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:22,817 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:24,840 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:24,861 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:26,401 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:26,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:26,886 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:26,897 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:27,563 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:28,921 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:28,933 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:30,955 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:30,966 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:31,942 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:32,955 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:32,992 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:33,007 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:35,073 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:35,089 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:37,205 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:37,258 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:37,956 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:38,939 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:08:39,359 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:39,384 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:40,043 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:08:40,044 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:08:40,044 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:08:40,045 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:08:40,945 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:08:41,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:41,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:41,946 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:43,535 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:08:43,589 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:43,617 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:43,622 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:45,687 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:45,718 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:47,799 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:47,829 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:47,948 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:48,864 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:49,877 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:49,914 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:51,290 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:08:51,292 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:08:51,292 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:08:51,292 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:08:51,950 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:08:51,950 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:51,977 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:51,998 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:53,951 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:54,035 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:54,056 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:54,294 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:08:55,952 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:56,099 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:56,113 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:56,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:08:56,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:08:56,683 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:08:56,684 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:08:56,684 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:08:56,684 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:08:56,953 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:08:57,953 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:08:58,146 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:58,161 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:08:59,685 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:00,225 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:00,246 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:02,272 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:02,293 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:04,333 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:04,352 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:04,686 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:06,376 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:06,397 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:08,421 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:08,435 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:08,941 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:09:09,942 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:10,460 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:10,473 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:11,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:11,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:11,961 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:09:12,554 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:12,573 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:15,558 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:16,329 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:09:16,794 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:16,821 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:18,910 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:18,951 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:20,559 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:21,046 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:21,070 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:23,093 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:23,115 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:23,549 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:09:23,550 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:09:23,551 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:09:23,551 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:09:23,966 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:09:23,966 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:09:25,191 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:25,213 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:25,968 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:09:26,398 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:26,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:26,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:27,365 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:27,387 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:29,497 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:29,516 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:31,618 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:31,650 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:31,971 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:09:31,994 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:33,717 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:33,740 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:34,638 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:09:34,639 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:09:34,640 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:09:34,640 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:09:34,972 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:09:35,785 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:35,802 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:35,973 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:09:37,641 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:37,854 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:37,884 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:38,944 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:09:39,956 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:39,975 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:09:39,981 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:41,026 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:09:41,027 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:09:41,027 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:09:41,028 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:09:41,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:41,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:41,976 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:09:41,976 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:09:42,007 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:42,027 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:43,570 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:44,047 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:44,056 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:46,076 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:46,089 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:48,570 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:50,050 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:09:50,082 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:50,106 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:51,981 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:09:53,419 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:53,447 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:53,709 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:55,501 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:55,510 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:56,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:09:56,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:09:57,571 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:57,588 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:59,545 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:09:59,632 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:09:59,653 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:01,674 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:01,693 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:03,715 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:03,730 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:04,546 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:05,263 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:10:05,265 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:10:05,265 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:10:05,265 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:10:05,762 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:05,781 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:05,987 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:10:05,987 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:10:07,813 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:07,840 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:07,988 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:10:08,946 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:10:09,948 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:11,146 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:11,164 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:11,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:11,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:13,212 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:13,234 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:13,990 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:10:15,260 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:15,276 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:15,520 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:17,327 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:17,358 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:18,208 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:10:18,210 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:10:18,210 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:10:18,210 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:10:18,992 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:10:19,388 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:19,993 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:10:20,394 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:21,211 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:21,994 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:10:22,782 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:22,793 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:24,151 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:10:24,152 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:10:24,152 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:10:24,717 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:10:24,995 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:10:25,996 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:10:26,398 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:26,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:26,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:26,723 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:10:26,748 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:26,759 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:27,997 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:10:28,781 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:28,791 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:30,810 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:30,821 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:31,526 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:32,843 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:32,854 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:34,886 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:34,899 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:36,527 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:38,164 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:38,190 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:38,949 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:10:40,233 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:40,254 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:41,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:41,402 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:41,623 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:42,299 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:42,315 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:44,354 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:44,369 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:46,415 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:46,427 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:46,623 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:48,796 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:10:48,798 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:10:48,798 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:10:48,798 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:10:49,008 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:10:49,588 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:49,613 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:50,008 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:10:51,641 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:51,654 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:51,799 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:53,678 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:53,699 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:55,722 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:55,735 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:56,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:10:56,403 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:10:57,569 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:10:58,975 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:10:58,998 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:00,013 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:11:02,718 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:11:03,112 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:03,139 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:03,158 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:03,997 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:11:03,998 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:11:03,998 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:11:03,998 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:11:04,015 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:11:05,281 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:05,303 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:06,015 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:11:07,363 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:07,387 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:08,951 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:08,952 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:11:09,474 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:09,516 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:10,017 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:11:10,858 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:11:10,859 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:11:10,859 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:11:10,860 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:11:11,017 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:11:11,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:11,403 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:11,599 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:11,620 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:12,018 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:11:13,736 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:13,772 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:14,019 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:11:14,828 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:15,854 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:15,883 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:17,995 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:18,037 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:19,830 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:20,115 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:20,141 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:22,320 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:22,348 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:24,486 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:24,522 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:24,830 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:26,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:26,403 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:26,585 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:26,621 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:28,654 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:28,690 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:30,560 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:30,715 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:30,738 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:32,055 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:11:32,056 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:11:32,056 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:11:32,057 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:11:33,028 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:11:33,369 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:33,398 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:34,028 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:11:36,057 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:37,386 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:11:37,418 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:37,433 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:38,955 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:11:39,461 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:39,474 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:41,399 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:41,402 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:41,403 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:41,516 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:41,533 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:43,629 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:43,651 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:45,770 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:45,795 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:46,595 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:47,911 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:47,936 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:50,036 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:11:50,062 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:50,083 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:52,167 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:52,220 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:52,253 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:54,305 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:54,340 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:54,430 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:11:54,431 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:11:54,431 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:11:54,432 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:11:55,038 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:11:56,039 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:11:56,403 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:11:56,403 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:11:56,403 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:56,440 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:57,601 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:11:58,521 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:11:58,539 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:00,041 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:00,585 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:00,601 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:01,464 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:01,465 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:12:01,466 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:12:01,466 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:12:02,042 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:12:02,042 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:02,641 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:02,652 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:03,467 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:04,043 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:04,685 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:04,696 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:06,718 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:06,729 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:08,628 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:08,956 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:12:10,804 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:12:10,839 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:10,863 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:11,403 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:11,403 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:12,901 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:12,915 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:13,815 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:13,949 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:13,951 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:12:13,951 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:12:13,951 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:12:14,047 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:12:14,946 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:14,964 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:16,048 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:16,988 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:17,001 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:19,023 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:19,039 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:19,149 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:22,310 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:22,331 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:24,150 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:24,354 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:24,369 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:26,389 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:26,401 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:26,403 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:26,403 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:28,427 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:28,439 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:29,545 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:30,477 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:30,494 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:32,521 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:33,833 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:34,546 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:35,858 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:35,878 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:37,917 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:37,940 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:38,959 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:12:39,960 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:39,966 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:39,976 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:41,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:41,404 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:42,014 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:42,037 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:45,682 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:45,972 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:12:46,004 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:46,025 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:46,063 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:48,064 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:48,067 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:48,099 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:49,846 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 21:12:49,847 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 21:12:49,853 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:49,915 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:49,917 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:50,065 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:50,065 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_78_6f247990e446903b7c69.wav +2024-05-13 21:12:50,065 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_78_ff389c0afad3d276441d.wav +2024-05-13 21:12:50,065 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_78_31d4a3506990aa644012.wav +2024-05-13 21:12:50,066 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_78_e8936f475da43789ec82.wav +2024-05-13 21:12:50,066 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_78_e0a9392514f4f0b15b68.wav +2024-05-13 21:12:50,066 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/table/predictions/eval-step-78_78_fc1d787b0060d14ab332.table.json +2024-05-13 21:12:50,066 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_78_ccbbfb2e4ad764ed2d56.wav +2024-05-13 21:12:50,066 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_78_2a1aa9c4aa9b28aeff3d.wav +2024-05-13 21:12:50,066 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/table/predictions +2024-05-13 21:12:50,066 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples +2024-05-13 21:12:50,140 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:50,172 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:50,563 INFO wandb-upload_6:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/0xl4bjcc-media/table/predictions/eval-step-78_78_fc1d787b0060d14ab332.table.json +2024-05-13 21:12:50,705 INFO wandb-upload_8:1264247 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmpph5ygv9b +2024-05-13 21:12:51,203 INFO wandb-upload_3:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/ab8u08t6-media/audio/Speech samples/eval_78_6f247990e446903b7c69.wav +2024-05-13 21:12:51,223 INFO wandb-upload_5:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/tgmdqye3-media/audio/Speech samples/eval_78_31d4a3506990aa644012.wav +2024-05-13 21:12:51,243 INFO wandb-upload_7:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/xcv02ij3-media/audio/Speech samples/eval_78_2a1aa9c4aa9b28aeff3d.wav +2024-05-13 21:12:51,248 INFO wandb-upload_0:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/gcdw87cb-media/audio/Speech samples/eval_78_e0a9392514f4f0b15b68.wav +2024-05-13 21:12:51,248 INFO wandb-upload_2:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/um1kf2m9-media/audio/Speech samples/eval_78_ff389c0afad3d276441d.wav +2024-05-13 21:12:51,266 INFO wandb-upload_1:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/wxgilkzi-media/audio/Speech samples/eval_78_e8936f475da43789ec82.wav +2024-05-13 21:12:51,354 INFO SenderThread:1264247 [sender.py:send_request_log_artifact():1455] logged artifact run-e3sq5zz5-predictionsevalstep78 - {'id': 'QXJ0aWZhY3Q6ODM0NzU5NDg3', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI0MzQ1Mg==', 'latestArtifact': None}} +2024-05-13 21:12:51,355 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:12:51,355 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-78_78_fc1d787b0060d14ab332.table.json with policy now +2024-05-13 21:12:51,356 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:12:51,356 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:51,356 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_78_6f247990e446903b7c69.wav with policy now +2024-05-13 21:12:51,356 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:12:51,356 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_78_2a1aa9c4aa9b28aeff3d.wav with policy now +2024-05-13 21:12:51,356 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:12:51,356 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_78_31d4a3506990aa644012.wav with policy now +2024-05-13 21:12:51,357 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:12:51,357 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_78_ccbbfb2e4ad764ed2d56.wav with policy now +2024-05-13 21:12:51,357 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:12:51,357 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_78_ff389c0afad3d276441d.wav with policy now +2024-05-13 21:12:51,357 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:12:51,357 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_78_e0a9392514f4f0b15b68.wav with policy now +2024-05-13 21:12:51,357 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:12:51,357 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_78_e8936f475da43789ec82.wav with policy now +2024-05-13 21:12:51,814 INFO wandb-upload_6:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/4lgyzjj6-media/table/predictions/eval-step-78_78_fc1d787b0060d14ab332.table.json +2024-05-13 21:12:52,066 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:52,210 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:52,220 INFO wandb-upload_4:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/w4k80bo9-media/audio/Speech samples/eval_78_ccbbfb2e4ad764ed2d56.wav +2024-05-13 21:12:52,236 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:52,443 INFO wandb-upload_8:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/3359o2au-media/audio/Speech samples/eval_78_ff389c0afad3d276441d.wav +2024-05-13 21:12:52,451 INFO wandb-upload_2:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/gon9mhp7-media/audio/Speech samples/eval_78_31d4a3506990aa644012.wav +2024-05-13 21:12:52,483 INFO wandb-upload_0:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/okszgqdt-media/audio/Speech samples/eval_78_e0a9392514f4f0b15b68.wav +2024-05-13 21:12:52,491 INFO wandb-upload_3:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/4qsczp5q-media/audio/Speech samples/eval_78_6f247990e446903b7c69.wav +2024-05-13 21:12:52,504 INFO wandb-upload_7:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/ba674zr4-media/audio/Speech samples/eval_78_2a1aa9c4aa9b28aeff3d.wav +2024-05-13 21:12:52,521 INFO wandb-upload_5:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/dljglj0l-media/audio/Speech samples/eval_78_e8936f475da43789ec82.wav +2024-05-13 21:12:53,371 INFO wandb-upload_1:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/hiuaz7mr-media/audio/Speech samples/eval_78_ccbbfb2e4ad764ed2d56.wav +2024-05-13 21:12:54,067 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:54,266 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:54,293 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:56,324 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:56,342 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:56,358 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:12:56,403 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:12:56,404 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:12:57,812 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:12:57,814 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:12:57,814 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:12:57,815 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:12:58,069 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:12:58,069 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:12:58,365 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:12:58,381 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:00,070 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:13:00,409 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:00,427 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:01,816 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:02,462 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:02,473 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:04,500 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:04,515 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:06,541 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:06,551 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:06,817 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:08,574 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:08,591 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:08,961 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:13:10,612 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:10,624 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:11,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:11,404 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:12,607 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:12,663 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:12,679 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:14,701 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:14,711 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:17,608 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:18,791 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:13:18,822 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:18,837 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:20,862 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:20,881 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:22,082 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:13:22,905 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:22,918 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:23,339 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:24,947 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:24,960 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:26,403 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:26,404 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:27,083 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:27,108 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:28,542 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:29,074 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:13:29,075 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:13:29,075 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:13:29,076 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:13:29,086 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:13:29,182 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:29,227 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:30,086 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:13:31,303 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:31,337 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:33,395 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:33,415 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:34,077 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:35,472 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:35,489 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:36,090 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:13:37,545 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:37,557 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:38,278 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:13:38,279 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:13:38,279 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:13:38,280 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:13:38,964 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:13:39,091 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:13:39,597 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:39,613 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:39,965 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:40,091 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:13:41,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:41,404 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:41,656 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:41,676 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:42,092 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:13:43,708 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:43,718 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:43,740 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:13:43,741 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:13:43,741 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:13:43,742 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:13:44,094 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:13:44,094 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:13:45,742 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:45,761 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:45,777 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:46,095 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:13:47,817 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:47,828 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:50,743 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:51,568 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:13:52,051 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:52,089 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:54,211 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:54,239 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:55,744 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:13:56,333 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:56,367 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:56,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:13:56,404 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:13:58,474 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:13:58,505 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:00,593 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:00,635 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:01,251 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:02,103 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:02,701 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:02,736 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:04,831 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:04,866 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:06,252 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:06,902 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:06,930 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:08,966 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:14:08,987 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:09,010 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:11,135 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:11,163 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:11,400 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:11,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:11,404 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:11,778 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:14:11,779 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:14:11,779 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:14:11,779 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:14:12,109 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:14:12,109 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:13,253 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:13,287 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:14,111 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:15,350 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:15,385 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:16,780 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:17,425 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:17,449 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:18,113 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:19,550 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:19,575 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:21,599 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:21,625 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:21,755 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:14:21,756 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:14:21,756 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:14:21,757 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:14:22,115 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:14:22,115 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:22,757 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:24,116 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:25,293 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:14:25,697 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:25,726 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:26,118 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:26,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:26,404 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:26,798 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:14:26,800 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:14:26,800 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:14:26,800 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:14:27,118 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:14:27,759 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:27,780 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:27,801 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:28,119 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:29,808 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:29,818 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:31,843 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:31,856 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:32,802 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:33,888 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:33,902 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:35,928 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:35,943 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:37,802 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:37,978 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:37,995 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:38,969 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:14:40,027 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:40,049 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:41,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:41,404 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:42,174 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:42,192 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:43,006 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:44,127 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:44,301 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:44,344 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:46,434 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:46,474 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:48,007 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:48,585 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:48,609 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:50,784 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:50,826 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:52,901 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:52,925 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:53,007 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:53,637 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:14:53,638 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:14:53,638 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:14:53,639 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:14:54,132 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:14:54,132 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:55,027 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:55,049 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:56,134 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:14:56,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:14:56,405 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:14:58,532 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:14:59,120 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:14:59,168 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:14:59,203 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:01,226 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:01,258 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:02,137 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:03,284 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:03,310 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:03,957 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:05,334 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:05,352 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:05,451 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:15:05,455 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:15:05,455 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:15:05,455 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:15:06,138 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:15:06,139 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:07,381 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:07,391 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:08,140 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:08,971 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:15:08,973 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:09,426 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:09,451 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:10,141 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:11,041 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:15:11,042 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:15:11,042 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:15:11,042 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:15:11,141 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:15:11,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:11,405 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:11,477 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:11,491 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:12,142 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:13,516 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:13,527 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:14,617 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:15,558 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:15,571 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:17,600 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:17,610 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:19,618 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:19,788 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:19,815 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:21,967 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:21,993 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:22,147 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:24,068 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:24,086 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:25,045 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:26,212 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:26,239 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:26,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:26,405 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:28,315 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:28,335 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:30,569 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:32,036 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:15:32,485 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:32,510 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:34,589 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:34,610 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:35,570 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:36,645 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:15:36,646 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:15:36,647 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:15:36,647 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:15:36,660 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:36,671 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:37,154 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:15:38,154 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:38,746 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:38,761 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:38,974 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:15:40,825 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:40,841 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:40,975 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:41,404 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:41,405 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:42,922 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:42,934 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:44,993 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:45,034 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:46,158 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:46,739 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:47,098 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:47,121 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:49,211 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:49,246 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:49,856 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:15:49,856 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:15:49,857 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:15:49,858 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:15:50,161 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:15:50,161 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:51,353 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:51,389 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:51,859 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:52,162 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:53,483 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:53,504 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:54,163 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:55,574 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:55,613 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:55,998 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:15:55,999 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:15:55,999 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:15:56,000 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:15:56,164 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:15:56,405 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:15:56,405 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:15:57,632 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:15:57,678 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:57,702 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:58,165 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:15:59,762 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:15:59,777 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:01,803 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:01,827 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:02,856 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:05,928 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:16:05,952 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:05,968 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:07,856 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:07,998 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:08,018 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:08,976 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:16:10,117 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:10,137 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:11,405 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:11,405 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:12,318 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:12,359 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:13,541 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:14,517 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:14,557 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:16,648 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:16,690 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:17,497 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:16:17,498 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:16:17,498 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:16:17,498 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:16:18,177 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:16:18,177 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:16:18,771 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:18,797 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:19,499 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:20,178 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:16:20,915 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:20,953 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:23,065 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:23,091 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:24,500 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:25,178 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:25,201 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:26,181 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:16:26,405 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:26,405 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:27,273 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:27,305 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:29,402 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:29,437 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:30,376 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:30,771 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:16:30,772 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:16:30,772 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:16:30,772 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:16:31,183 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:16:31,540 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:31,563 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:32,183 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:16:33,622 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:33,659 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:35,423 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:35,682 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:35,702 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:36,186 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:16:37,187 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:16:37,189 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:16:37,189 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:16:37,189 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:16:38,191 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:16:38,191 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:16:38,978 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:16:39,702 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:16:39,748 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:39,767 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:40,192 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:16:41,127 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:41,405 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:41,405 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:41,792 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:41,810 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:43,837 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:43,853 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:45,911 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:45,925 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:46,618 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:48,139 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:48,163 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:50,312 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:50,341 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:51,618 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:52,429 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:52,455 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:54,571 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:54,595 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:56,405 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:16:56,405 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:16:56,626 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:16:56,711 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:56,732 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:56,778 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:16:56,779 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:16:56,779 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:16:56,780 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:16:57,200 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:16:58,201 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:16:58,843 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:16:58,872 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:00,966 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:00,993 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:01,781 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:03,070 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:03,096 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:05,156 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:05,181 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:07,240 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:07,260 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:07,768 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:08,206 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:17:08,980 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:17:09,287 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:09,303 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:11,405 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:11,406 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:13,031 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:13,109 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:17:13,110 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:17:13,110 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:17:13,111 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:17:13,209 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:17:13,431 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:17:13,460 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:13,474 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:14,210 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:17:15,558 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:15,609 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:17,680 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:17,711 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:18,212 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:17:18,531 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:19,765 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:19,788 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:20,814 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:17:20,815 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:17:20,816 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:17:20,816 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:17:21,214 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:17:21,833 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:21,854 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:22,214 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:17:23,700 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:23,875 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:23,894 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:24,216 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:17:25,916 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:25,936 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:26,217 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:17:26,405 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:26,406 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:27,963 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:27,980 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:29,622 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:30,008 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:30,022 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:33,252 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:33,295 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:34,624 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:35,354 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:35,381 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:37,438 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:37,449 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:38,982 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:17:39,509 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:39,521 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:39,984 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:41,405 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:41,406 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:41,565 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:41,583 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:43,609 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:43,623 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:45,609 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:47,483 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:17:47,529 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:47,555 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:50,609 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:50,814 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:50,836 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:52,888 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:52,908 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:54,232 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:17:54,951 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:54,972 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:56,214 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:17:56,233 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:17:56,406 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:17:56,406 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:17:57,052 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:57,080 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:17:58,126 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: log_artifact +2024-05-13 21:17:58,126 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: log_artifact +2024-05-13 21:17:58,132 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:17:58,191 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:17:58,193 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:17:58,234 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:17:58,234 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_117_dd8bdcb385062c83d9db.wav +2024-05-13 21:17:58,234 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_117_51c4feb7b25049501e37.wav +2024-05-13 21:17:58,234 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/table/predictions/eval-step-117_116_f8e1efdb2e6a51f3de3f.table.json +2024-05-13 21:17:58,235 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_117_6d818be44006fe311b25.wav +2024-05-13 21:17:58,235 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_117_855fad46eee4554f15a7.wav +2024-05-13 21:17:58,235 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_117_ea76ce7bbfab47610005.wav +2024-05-13 21:17:58,235 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_117_7633a61e6ee80a024fb2.wav +2024-05-13 21:17:58,235 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_created():271] file/dir created: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples/eval_117_3f7a73b6eb939439853b.wav +2024-05-13 21:17:58,235 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/table/predictions +2024-05-13 21:17:58,235 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/media/audio/Speech samples +2024-05-13 21:17:58,750 INFO wandb-upload_0:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/zu4jghwc-media/table/predictions/eval-step-117_116_f8e1efdb2e6a51f3de3f.table.json +2024-05-13 21:17:59,150 INFO wandb-upload_6:1264247 [upload_job.py:push():88] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmptu914sby +2024-05-13 21:17:59,366 INFO wandb-upload_4:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/n60b03rm-media/audio/Speech samples/eval_117_dd8bdcb385062c83d9db.wav +2024-05-13 21:17:59,411 INFO wandb-upload_8:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/irshxbdf-media/audio/Speech samples/eval_117_51c4feb7b25049501e37.wav +2024-05-13 21:17:59,414 INFO wandb-upload_7:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/kbtr4twh-media/audio/Speech samples/eval_117_6d818be44006fe311b25.wav +2024-05-13 21:17:59,415 INFO wandb-upload_1:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/oqa7enbg-media/audio/Speech samples/eval_117_855fad46eee4554f15a7.wav +2024-05-13 21:17:59,418 INFO wandb-upload_2:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/sq39d93s-media/audio/Speech samples/eval_117_ea76ce7bbfab47610005.wav +2024-05-13 21:17:59,427 INFO wandb-upload_5:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/bp076tbx-media/audio/Speech samples/eval_117_7633a61e6ee80a024fb2.wav +2024-05-13 21:17:59,473 INFO wandb-upload_3:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/sexley1k-media/audio/Speech samples/eval_117_3f7a73b6eb939439853b.wav +2024-05-13 21:17:59,813 INFO SenderThread:1264247 [sender.py:send_request_log_artifact():1455] logged artifact run-e3sq5zz5-predictionsevalstep117 - {'id': 'QXJ0aWZhY3Q6ODM0NzY0MTg2', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE3NDI0NDc2MQ==', 'latestArtifact': None}} +2024-05-13 21:17:59,813 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:17:59,813 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/table/predictions/eval-step-117_116_f8e1efdb2e6a51f3de3f.table.json with policy now +2024-05-13 21:17:59,814 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:17:59,814 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:17:59,814 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:17:59,815 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:17:59,815 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_117_855fad46eee4554f15a7.wav with policy now +2024-05-13 21:17:59,815 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:17:59,815 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_117_3f7a73b6eb939439853b.wav with policy now +2024-05-13 21:17:59,815 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:17:59,815 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_117_6d818be44006fe311b25.wav with policy now +2024-05-13 21:17:59,815 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:17:59,815 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_117_7633a61e6ee80a024fb2.wav with policy now +2024-05-13 21:17:59,817 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:17:59,818 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_117_dd8bdcb385062c83d9db.wav with policy now +2024-05-13 21:17:59,818 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:17:59,818 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_117_ea76ce7bbfab47610005.wav with policy now +2024-05-13 21:17:59,818 DEBUG SenderThread:1264247 [sender.py:send():378] send: files +2024-05-13 21:17:59,818 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file media/audio/Speech samples/eval_117_51c4feb7b25049501e37.wav with policy now +2024-05-13 21:18:00,236 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:18:00,236 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:18:00,263 INFO wandb-upload_0:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/80mj24jb-media/table/predictions/eval-step-117_116_f8e1efdb2e6a51f3de3f.table.json +2024-05-13 21:18:00,472 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:00,496 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:00,924 INFO wandb-upload_2:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/gpm6jfp7-media/audio/Speech samples/eval_117_6d818be44006fe311b25.wav +2024-05-13 21:18:00,951 INFO wandb-upload_3:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/umhwo3ks-media/audio/Speech samples/eval_117_7633a61e6ee80a024fb2.wav +2024-05-13 21:18:00,952 INFO wandb-upload_7:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/wv9y7857-media/audio/Speech samples/eval_117_3f7a73b6eb939439853b.wav +2024-05-13 21:18:00,953 INFO wandb-upload_4:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/yoka25j6-media/audio/Speech samples/eval_117_855fad46eee4554f15a7.wav +2024-05-13 21:18:00,954 INFO wandb-upload_8:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/0p47usye-media/audio/Speech samples/eval_117_dd8bdcb385062c83d9db.wav +2024-05-13 21:18:00,972 INFO wandb-upload_1:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/on7ietqa-media/audio/Speech samples/eval_117_51c4feb7b25049501e37.wav +2024-05-13 21:18:01,005 INFO wandb-upload_5:1264247 [upload_job.py:push():130] Uploaded file /tmp/tmpjbmu0jewwandb/uiawnrua-media/audio/Speech samples/eval_117_ea76ce7bbfab47610005.wav +2024-05-13 21:18:01,819 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:02,237 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:18:02,625 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:02,659 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:04,740 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:04,771 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:06,815 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:06,820 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:06,875 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:08,894 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:08,907 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:08,985 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:18:10,932 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:10,945 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:11,406 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:11,406 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:12,565 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:12,721 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:18:12,722 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:18:12,723 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:18:12,723 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:18:12,968 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:12,986 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:13,242 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:18:14,243 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:18:15,008 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:15,019 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:17,039 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:17,052 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:17,724 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:19,075 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:19,089 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:20,242 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: partial_history +2024-05-13 21:18:20,243 DEBUG SenderThread:1264247 [sender.py:send():378] send: history +2024-05-13 21:18:20,243 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: summary_record +2024-05-13 21:18:20,246 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/wandb-summary.json +2024-05-13 21:18:20,246 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:18:20,393 INFO SenderThread:1264247 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-05-13 21:18:22,248 INFO Thread-12 :1264247 [dir_watcher.py:_on_file_modified():288] file/dir modified: /raid/sanchit/parler-tts-mini-v0.1-expresso-concatenated/wandb/run-20240513_205708-e3sq5zz5/files/output.log +2024-05-13 21:18:22,761 ERROR gpu :1264247 [interfaces.py:aggregate():159] Failed to serialize metric: division by zero +2024-05-13 21:18:22,794 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:23,393 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:24,046 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:26,071 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:26,087 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:26,406 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: stop_status +2024-05-13 21:18:26,406 DEBUG SenderThread:1264247 [sender.py:send_request():405] send_request: stop_status +2024-05-13 21:18:28,110 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:28,122 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:28,538 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:30,151 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:30,163 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:32,192 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:32,204 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:33,539 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:34,237 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:34,259 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:36,280 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:36,298 ERROR gpu :1264247 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-05-13 21:18:38,539 DEBUG HandlerThread:1264247 [handler.py:handle_request():158] handle_request: status_report +2024-05-13 21:18:38,987 DEBUG SenderThread:1264247 [sender.py:send():378] send: stats +2024-05-13 21:18:39,096 WARNING FileStreamThread:1264247 [file_stream.py:request_with_retry():674] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/sanchit-gandhi/parler-speech/e3sq5zz5/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/sanchit-gandhi/parler-speech/e3sq5zz5/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"train/loss": 4.717447757720947, "train/time": 835.6318266391754, "train/epoch": 2.9230769230769234, "train/learning_rate": 3.648e-05, "_timestamp": 1715627833.1085007, "_runtime": 1204.740018606186, "_step": 114, "predictions/eval-step-39": {"_type": "table-file", "sha256": "a379d6cee0305d3840486549a52815b229ed541ac8d3323c3079a8cb5b5f5822", "size": 1616, "artifact_path": "wandb-client-artifact://51ntdbwieaph12sdvciuzbfjbu3aj6jimzn901ltron6by1imkuxghb4f7phqtupvl954guyrzex5ebdjeolc2791l6iu32y0lykmcjaj2oyr7i7ioxaon7uhzzi55nc/predictions/eval-step-39.table.json", "_latest_artifact_path": "wandb-client-artifact://m1nnvusjl9on0bk0gmucqg9zynhflk4nqnkl7lecvm1t8ejt2fzum29112sm0ijkhue700bh2bwqau84o9anj8ijnqr3addksuujkp2b2zykfrb0wiqc8h0t9zu5k6b4:latest/predictions/eval-step-39.table.json", "path": "media/table/predictions/eval-step-39_38_a379d6cee0305d384048.table.json", "ncols": 3, "nrows": 7}, "Speech samples/eval": {"_type": "audio", "count": 7, "audio": [{"_type": "audio-file", "sha256": "6f247990e446903b7c694faf9746df9556ac8414f7834f644130de678d8f248d", "size": 1082412, "path": "media/audio/Speech samples/eval_78_6f247990e446903b7c69.wav", "caption": "So, shall we eat? His plane exploded in midair. --- DESCRIPTION: Elisabeth speaks slightly slowly with high quality."}, {"_type": "audio-file", "sha256": "2a1aa9c4aa9b28aeff3d5b10be80348e8ec11fa2a20124c5bb79ed5bdab169f7", "size": 1082412, "path": "media/audio/Speech samples/eval_78_2a1aa9c4aa9b28aeff3d.wav", "caption": "Can\'t even read a map! --- DESCRIPTION: Thomas speaks sad and slightly slowly with high quality."}, {"_type": "audio-file", "sha256": "31d4a3506990aa6440125ae50eb4b35ee52557797637c467607d2540d42bf6fc", "size": 1082412, "path": "media/audio/Speech samples/eval_78_31d4a3506990aa644012.wav", "caption": "*Blue Origin\'s* rocket takes off and lands vertically. --- DESCRIPTION: Thomas speaks at a moderate speed with high quality."}, {"_type": "audio-file", "sha256": "ccbbfb2e4ad764ed2d56b2cbcd87060666aaa21ba11a644c0ab03a7c14c9b531", "size": 1082412, "path": "media/audio/Speech samples/eval_78_ccbbfb2e4ad764ed2d56.wav", "caption": "What percentage of the car vandalizing population would have any interest in Medieval art? --- DESCRIPTION: Talia speaks happily with quite fast delivery and high quality."}, {"_type": "audio-file", "sha256": "ff389c0afad3d276441dcaac8e379bceba631030e3d91d9af965a04c1ac6bfdb", "size": 1082412, "path": "media/audio/Speech samples/eval_78_ff389c0afad3d276441d.wav", "caption": "And Noom helps me manage that. Strawberry ice cream sundae, right? I just wasn\'t able to talk him out of it. --- DESCRIPTION: Talia speaks with a moderate pace and sad tone with high quality audio."}, {"_type": "audio-file", "sha256": "e0a9392514f4f0b15b689cf75ef0fad637aed13e007af951f95a479756c0dda4", "size": 1082412, "path": "media/audio/Speech samples/eval_78_e0a9392514f4f0b15b68.wav", "caption": "Yes, I love the thrift shops. --- DESCRIPTION: Talia speaks with clear and deliberate enunciation and quite slowly with high quality."}, {"_type": "audio-file", "sha256": "e8936f475da43789ec829bf60f86fe32de7acd242165490d09cb3252ea1269c9", "size": 1082412, "path": "media/audio/Speech samples/eval_78_e8936f475da43789ec82.wav", "caption": "Figure out a way to like really duct tape around it so no water can get in. So I had to get myself back in that emotional place and get sad again. I have thoughts and they\'re like, \\"I suck at this\\". --- DESCRIPTION: Jerry speaks in a whisper at a moderate speed with high quality."}], "sampleRates": [44100, 44100, 44100, 44100, 44100, 44100, 44100], "durations": [12.271746031746032, 12.271746031746032, 12.271746031746032, 12.271746031746032, 12.271746031746032, 12.271746031746032, 12.271746031746032], "captions": ["So, shall we eat? His plane exploded in midair. --- DESCRIPTION: Elisabeth speaks slightly slowly with high quality.", "Can\'t even read a map! --- DESCRIPTION: Thomas speaks sad and slightly slowly with high quality.", "*Blue Origin\'s* rocket takes off and lands vertically. --- DESCRIPTION: Thomas speaks at a moderate speed with high quality.", "What percentage of the car vandalizing population would have any interest in Medieval art? --- DESCRIPTION: Talia speaks happily with quite fast delivery and high quality.", "And Noom helps me manage that. Strawberry ice cream sundae, right? I just wasn\'t able to talk him out of it. --- DESCRIPTION: Talia speaks with a moderate pace and sad tone with high quality audio.", "Yes, I love the thrift shops. --- DESCRIPTION: Talia speaks with clear and deliberate enunciation and quite slowly with high quality.", "Figure out a way to like really duct tape around it so no water can get in. So I had to get myself back in that emotional place and get sad again. I have thoughts and they\'re like, \\"I suck at this\\". --- DESCRIPTION: Jerry speaks in a whisper at a moderate speed with high quality."]}, "eval/loss": 4.9100728034973145, "eval/clap": 0.3258473575115204, "eval/wer": 60.0, "eval/time": 30.96561551094055, "eval/epoch": 2.0, "predictions/eval-step-78": {"_type": "table-file", "sha256": "fc1d787b0060d14ab3322d3b077cedfcdb521ef9e8c24f68149505def619d7a1", "size": 1602, "artifact_path": "wandb-client-artifact://jxn59yor6jtrdqerwzs5x6hojj6w3cfqwcfw02ytcb9zwad74i6b1umm4qcfglqqvowfc5z6qotvjsegf9k1wsgqbn2rs7qmk7esbwy8af1zj8kw5rsipnjhvcec6oll/predictions/eval-step-78.table.json", "_latest_artifact_path": "wandb-client-artifact://7ybp3f7hdd36j0kgbix0w0m68c5ahy2exypp8zb1o6cvr4liayg2qcisjp37tfms1wyslja12rkp55a2vdufzv8b3dw2szuk83d6ahc6wlyg7tm49oyhrty8ipfxnsvo:latest/predictions/eval-step-78.table.json", "path": "media/table/predictions/eval-step-78_78_fc1d787b0060d14ab332.table.json", "ncols": 3, "nrows": 7}}']}, 'wandb-history.jsonl': {'offset': 56, 'content': ['{"train/loss": 4.340236186981201, "train/time": 819.3006064891815, "train/epoch": 2.871794871794872, "train/learning_rate": 3.584e-05, "_timestamp": 1715627816.7772927, "_runtime": 1188.4088106155396, "_step": 112}', '{"train/loss": 4.717447757720947, "train/time": 835.6318266391754, "train/epoch": 2.9230769230769234, "train/learning_rate": 3.648e-05, "_timestamp": 1715627833.1085007, "_runtime": 1204.740018606186, "_step": 114}']}, 'wandb-events.jsonl': {'offset': 39, 'content': ['{"system.proc.memory.availableMB": 409562.75, "system.memory": 20.6, "system.proc.memory.rssMB": 7100.5, "system.proc.memory.percent": 1.38, "system.network.sent": 173581712.13, "system.network.recv": 105497802.33, "system.cpu": 0.78, "system.cpu.0.cpu_percent": 18.88, "system.cpu.1.cpu_percent": 2.53, "system.cpu.2.cpu_percent": 0.7, "system.cpu.3.cpu_percent": 0.2, "system.cpu.4.cpu_percent": 2.1, "system.cpu.5.cpu_percent": 0.33, "system.cpu.6.cpu_percent": 3.27, "system.cpu.7.cpu_percent": 3.08, "system.cpu.8.cpu_percent": 0.5, "system.cpu.9.cpu_percent": 1.43, "system.cpu.10.cpu_percent": 0.03, "system.cpu.11.cpu_percent": 0.3, "system.cpu.12.cpu_percent": 43.35, "system.cpu.13.cpu_percent": 8.51, "system.cpu.14.cpu_percent": 4.33, "system.cpu.15.cpu_percent": 2.26, "system.cpu.16.cpu_percent": 0.0, "system.cpu.17.cpu_percent": 3.13, "system.cpu.18.cpu_percent": 0.0, "system.cpu.19.cpu_percent": 0.2, "system.cpu.20.cpu_percent": 7.45, "system.cpu.21.cpu_percent": 5.89, "system.cpu.22.cpu_percent": 5.31, "system.cpu.23.cpu_percent": 0.4, "system.cpu.24.cpu_percent": 0.0, "system.cpu.25.cpu_percent": 44.58, "system.cpu.26.cpu_percent": 0.17, "system.cpu.27.cpu_percent": 0.0, "system.cpu.28.cpu_percent": 0.07, "system.cpu.29.cpu_percent": 3.1, "system.cpu.30.cpu_percent": 0.0, "system.cpu.31.cpu_percent": 5.51, "system.cpu.32.cpu_percent": 24.9, "system.cpu.33.cpu_percent": 4.15, "system.cpu.34.cpu_percent": 0.57, "system.cpu.35.cpu_percent": 1.56, "system.cpu.36.cpu_percent": 0.57, "system.cpu.37.cpu_percent": 0.13, "system.cpu.38.cpu_percent": 0.3, "system.cpu.39.cpu_percent": 0.1, "system.cpu.40.cpu_percent": 0.0, "system.cpu.41.cpu_percent": 2.7, "system.cpu.42.cpu_percent": 0.07, "system.cpu.43.cpu_percent": 0.8, "system.cpu.44.cpu_percent": 3.72, "system.cpu.45.cpu_percent": 0.27, "system.cpu.46.cpu_percent": 0.6, "system.cpu.47.cpu_percent": 4.29, "system.cpu.48.cpu_percent": 2.03, "system.cpu.49.cpu_percent": 0.07, "system.cpu.50.cpu_percent": 0.07, "system.cpu.51.cpu_percent": 0.07, "system.cpu.52.cpu_percent": 43.72, "system.cpu.53.cpu_percent": 56.33, "system.cpu.54.cpu_percent": 1.3, "system.cpu.55.cpu_percent": 0.13, "system.cpu.56.cpu_percent": 3.59, "system.cpu.57.cpu_percent": 0.43, "system.cpu.58.cpu_percent": 0.23, "system.cpu.59.cpu_percent": 4.84, "system.cpu.60.cpu_percent": 2.9, "system.cpu.61.cpu_percent": 5.8, "system.cpu.62.cpu_percent": 0.1, "system.cpu.63.cpu_percent": 1.57, "system.cpu.64.cpu_percent": 0.1, "system.cpu.65.cpu_percent": 7.57, "system.cpu.66.cpu_percent": 0.1, "system.cpu.67.cpu_percent": 0.23, "system.cpu.68.cpu_percent": 2.67, "system.cpu.69.cpu_percent": 1.4, "system.cpu.70.cpu_percent": 0.57, "system.cpu.71.cpu_percent": 2.37, "system.cpu.72.cpu_percent": 23.47, "system.cpu.73.cpu_percent": 5.23, "system.cpu.74.cpu_percent": 3.01, "system.cpu.75.cpu_percent": 0.13, "system.cpu.76.cpu_percent": 0.03, "system.cpu.77.cpu_percent": 0.07, "system.cpu.78.cpu_percent": 0.07, "system.cpu.79.cpu_percent": 0.17, "system.cpu.80.cpu_percent": 99.4, "system.cpu.81.cpu_percent": 0.0, "system.cpu.82.cpu_percent": 0.1, "system.cpu.83.cpu_percent": 0.67, "system.cpu.84.cpu_percent": 0.13, "system.cpu.85.cpu_percent": 1.8, "system.cpu.86.cpu_percent": 5.67, "system.cpu.87.cpu_percent": 3.59, "system.cpu.88.cpu_percent": 55.33, "system.cpu.89.cpu_percent": 0.03, "system.cpu.90.cpu_percent": 0.0, "system.cpu.91.cpu_percent": 0.03, "system.cpu.92.cpu_percent": 4.63, "system.cpu.93.cpu_percent": 2.92, "system.cpu.94.cpu_percent": 2.01, "system.cpu.95.cpu_percent": 5.1, "system.cpu.96.cpu_percent": 0.03, "system.cpu.97.cpu_percent": 0.07, "system.cpu.98.cpu_percent": 0.1, "system.cpu.99.cpu_percent": 1.51, "system.cpu.100.cpu_percent": 0.13, "system.cpu.101.cpu_percent": 0.4, "system.cpu.102.cpu_percent": 0.53, "system.cpu.103.cpu_percent": 0.67, "system.cpu.104.cpu_percent": 0.13, "system.cpu.105.cpu_percent": 0.1, "system.cpu.106.cpu_percent": 0.2, "system.cpu.107.cpu_percent": 0.07, "system.cpu.108.cpu_percent": 1.37, "system.cpu.109.cpu_percent": 0.2, "system.cpu.110.cpu_percent": 1.29, "system.cpu.111.cpu_percent": 1.77, "system.cpu.112.cpu_percent": 2.37, "system.cpu.113.cpu_percent": 0.07, "system.cpu.114.cpu_percent": 0.07, "system.cpu.115.cpu_percent": 2.13, "system.cpu.116.cpu_percent": 0.07, "system.cpu.117.cpu_percent": 0.17, "system.cpu.118.cpu_percent": 0.13, "system.cpu.119.cpu_percent": 0.13, "system.cpu.120.cpu_percent": 1.9, "system.cpu.121.cpu_percent": 1.77, "system.cpu.122.cpu_percent": 0.13, "system.cpu.123.cpu_percent": 4.0, "system.cpu.124.cpu_percent": 0.03, "system.cpu.125.cpu_percent": 1.43, "system.cpu.126.cpu_percent": 3.57, "system.cpu.127.cpu_percent": 1.4, "system.proc.cpu.threads": 15, "system.disk.\\\\.usagePercent": 99.7, "system.disk.\\\\.usageGB": 1663.51, "system.disk.in": 28.63, "system.disk.out": 12863.52, "system.gpu.0.memoryAllocated": 85.43, "system.gpu.1.memoryAllocated": 80.6, "system.gpu.2.memoryAllocated": 59.42, "system.gpu.3.memoryAllocated": 4.69, "system.gpu.4.memoryAllocated": 66.53, "system.gpu.process.4.memoryAllocated": 66.53, "system.gpu.0.memoryAllocatedBytes": 73387802624.0, "system.gpu.1.memoryAllocatedBytes": 69235507200.0, "system.gpu.2.memoryAllocatedBytes": 51038388224.0, "system.gpu.3.memoryAllocatedBytes": 201588736.0, "system.gpu.4.memoryAllocatedBytes": 57147392000.0, "system.gpu.process.4.memoryAllocatedBytes": 57147392000.0, "system.gpu.0.memory": 0.0, "system.gpu.1.memory": 50.2, "system.gpu.2.memory": 47.6, "system.gpu.3.memory": 0.0, "system.gpu.4.memory": 53.07, "system.gpu.process.4.memory": 53.07, "system.gpu.0.gpu": 0.0, "system.gpu.1.gpu": 98.8, "system.gpu.2.gpu": 97.27, "system.gpu.3.gpu": 0.0, "system.gpu.4.gpu": 99.0, "system.gpu.process.4.gpu": 99.0, "system.gpu.0.temp": 71.0, "system.gpu.1.temp": 79.93, "system.gpu.2.temp": 78.13, "system.gpu.3.temp": 40.0, "system.gpu.4.temp": 79.67, "system.gpu.process.4.temp": 79.67, "_wandb": true, "_timestamp": 1715627858.979598, "_runtime": 1230.611116}']}, 'output.log': {'offset': 2323, 'content': ['2024-05-13T19:17:13.775113 Step... (114 / 312 | Loss: 4.717447757720947, Learning Rate: 3.648e-05)\n', '2024-05-13T19:17:21.776881 Step... (116 / 312 | Loss: 4.568950653076172, Learning Rate: 3.7120000000000004e-05)\n', 'ERROR 2024-05-13T19:17:26.028664 Evaluating - Inference ...: 100%|█████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 2.13it/s]\n', 'ERROR 2024-05-13T19:17:26.028664 Evaluating - Generation ...: 0%| | 0/1 [00:00