diff --git "a/log.txt" "b/log.txt" new file mode 100644--- /dev/null +++ "b/log.txt" @@ -0,0 +1,1472 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[2023-11-15 11:18:25,500] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,746] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,760] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,924] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,932] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,960] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,963] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,964] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,229] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,037] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,116] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,361] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,390] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,403] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,407] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:25,408] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2023-11-15 11:18:41,275] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,108] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,216] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,440] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,480] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,480] [INFO] [comm.py:668:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +[2023-11-15 11:18:45,514] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,543] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,545] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,568] [INFO] [comm.py:637:init_distributed] cdb=None +it_distributed] cdb=None +[2023-11-15 11:18:45,326] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,373] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,381] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,392] [INFO] [comm.py:637:init_distributed] cdb=None +[2023-11-15 11:18:45,420] [INFO] [comm.py:637:init_distributed] cdb=None +11/15/2023 11:18:47 - WARNING - __main__ - Process rank: 1, device: cuda:1, n_gpu: 1distributed training: True, 16-bits training: False +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +11/15/2023 11:18:47 - WARNING - __main__ - Process rank: 7, device: cuda:7, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:18:47 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: False +11/1/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +11/15/2023 11:18:47 - WARNING - __main__ - Process rank: 5, device: cuda:5, n_gpu: 1distributed training: True, 16-bits training: False +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +11/15/2023 11:18:47 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:18:47 - INFO - __main__ - Training/evaluation parameters TrainingArguments( +_n_gpu=1, +adafactor=False, +adam_beta1=0.9, +adam_beta2=0.999, +adam_epsilon=1e-08, +auto_find_batch_size=False, +bf16=True, +bf16_full_eval=True, +data_seed=None, +dataloader_drop_last=False, +dataloader_num_workers=8, +dataloader_pin_memory=True, +ddp_bucket_cap_mb=None, +ddp_find_unused_parameters=None, +ddp_timeout=72000, +debug=[], +deepspeed=train/deepspeed_config_zero2.json, +disable_tqdm=False, +do_eval=False, +do_predict=False, +do_train=True, +eval_accumulation_steps=None, +eval_delay=0, +eval_steps=None, +evaluation_strategy=no, +fp16=False, +fp16_backend=auto, +fp16_full_eval=False, +fp16_opt_level=O1, +fsdp=[], +fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, +fsdp_min_num_params=0, +fsdp_transformer_layer_cls_to_wrap=None, +full_determinism=False, +gradient_accumulation_steps=1, +gradient_checkpointing=True, +greater_is_better=None, +group_by_length=False, +half_precision_backend=auto, +hub_model_id=None, +hub_private_repo=False, +hub_strategy=every_save, +hub_token=, +ignore_data_skip=False, +include_inputs_for_metrics=False, +jit_mode_eval=False, +label_names=None, +label_smoothing_factor=0.0, +learning_rate=2e-05, +length_column_name=length, +load_best_model_at_end=False, +local_rank=0, +log_level=passive, +log_level_replica=warning, +log_on_each_node=True, +logging_dir=/apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20/runs/Nov15_11-18-24_ts-cbba87c5e7504a249f5127103d9ce40f-worker-1, +logging_first_step=False, +logging_nan_inf_filter=True, +logging_steps=10, +logging_strategy=steps, +lr_scheduler_type=cosine, +max_grad_norm=1.0, +max_steps=-1, +metric_for_best_model=None, +mp_parameters=, +no_cuda=False, +num_train_epochs=3.0, +optim=adamw_hf, +optim_args=None, +output_dir=/apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20, +overwrite_output_dir=True, +past_index=-1, +per_device_eval_batch_size=2, +per_device_train_batch_size=48, +prediction_loss_only=False, +push_to_hub=False, +push_to_hub_model_id=None, +push_to_hub_organization=None, +push_to_hub_token=, +ray_scope=last, +remove_unused_columns=True, +report_to=['tensorboard'], +resume_from_checkpoint=None, +run_name=/apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20, +save_on_each_node=False, +save_steps=500, +save_strategy=steps, +save_total_limit=2, +seed=34, +sharded_ddp=[], +skip_memory_metrics=True, +tf32=None, +torch_compile=False, +torch_compile_backend=None, +torch_compile_mode=None, +torchdynamo=None, +tpu_metrics_debug=False, +tpu_num_cores=None, +use_ipex=False, +use_legacy_prediction_loop=False, +use_mps_device=False, +warmup_ratio=0.03, +warmup_steps=0, +weight_decay=0.0, +xpu_backend=None, +) +11/15/2023 11:18:47 - INFO - __main__ - Loading dataset from file: {'train': '/apdcephfs/share_733425/vinnylywang/jianhuipang/datasets/wmt23/wmt23-deen/train.catalpacagpt4.newstests17to20.10000.de2en.hf.json', 'validation': '/apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/test/WMT22/newstest22.de2en.hf.json'} +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in fav11/15/2023 11:18:47 - WARNING - __main__ - Process rank: 2, device: cuda:2, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:18:47 - WARNING - __main__ - Process rank: 5, device: cuda:5, n_gpu: 1distributed training: True, 16-bits training: False +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +11/15/2023 11:18:47 - WARNING - __main__ - Process rank: 4, device: cuda:4, n_gpu: 1distributed training: True, 16-bits training: False +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +11/15/2023 11:18:47 - WARNING - __main__ - Process rank: 6, device: cuda:6, n_gpu: 1distributed training: True, 16-bits training: False +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +Using custom data configuration default-b62333a3cd00a31b +11/15/2023 11:18:47 - INFO - datasets.builder - Using custom data configuration default-b62333a3cd00a31b +Loading Dataset Infos from /apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/packaged_modules/json +11/15/2023 11:18:47 - INFO - datasets.info - Loading Dataset Infos from /apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/packaged_modules/json +Overwrite dataset info from restored data version if exists. +11/15/2023 11:18:47 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists. +Loading Dataset info from /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96 +11/15/2023 11:18:47 - INFO - datasets.info - Loading Dataset info from /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96 +Found cached dataset json (/apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +11/15/2023 11:18:47 - INFO - datasets.builder - Found cached dataset json (/apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Loading Dataset info from /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96 +11/15/2023 11:18:47 - INFO - datasets.info - Loading Dataset info from /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96 +[INFO|configuration_utils.py:666] 2023-11-15 11:18:47,928 >> loading configuration file /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf/config.json +[INFO|configuration_utils.py:720] 2023-11-15 11:18:47,929 >> Model config LlamaConfig { + "_name_or_path": "/apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf", + "architectures": [ + "LlamaForCausalLM" + ], + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 11008, + "max_position_embeddings": 4096, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 32, + "pad_token_id": 0, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.28.0.dev0", + "use_cache": true, + "vocab_size": 32000 +} + +11/15/2023 11:18:47 - INFO - __main__ - Tokenizer_kwargs: {'cache_dir': None, 'use_fast': True, 'revision': 'main', 'use_auth_token': None} +[INFO|tokenization_utils_base.py:1801] 2023-11-15 11:18:47,936 >> loading file tokenizer.model +[INFO|tokenization_utils_base.py:1801] 2023-11-15 11:18:47,936 >> loading file added_tokens.json +[INFO|tokenization_utils_base.py:1801] 2023-11-15 11:18:47,936 >> loading file special_tokens_map.json +[INFO|tokenization_utils_base.py:1801] 2023-11-15 11:18:47,936 >> loading file tokenizer_config.json +11/15/2023 11:18:47 - INFO - __main__ - Loading checkpoints in dtype: None +[INFO|modeling_utils.py:2395] 2023-11-15 11:18:47,960 >> loading weights file /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf/pytorch_model.bin.index.json +[INFO|configuration_utils.py:575] 2023-11-15 11:18:47,961 >> Generate config GenerationConfig { + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "transformers_version": "4.28.0.dev0" +} + +ion_utils_base.py:1801] 2023-11-15 11:18:47,900 >> loading file tokenizer_config.json +11/15/2023 11:18:47 - INFO - __main__ - Loading checkpoints in dtype: None +[INFO|modeling_utils.py:2395] 2023-11-15 11:18:47,925 >> loading weights file /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf/pytorch_model.bin.index.json +[INFO|configuration_utils.py:575] 2023-11-15 11:18:47,928 >> Generate config GenerationConfig { + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "transformers_version": "4.28.0.dev0" +} + + Loading checkpoint shards: 0%| | 0/2 [00:00> Using pad_token, but it is not set yet. +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,327 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.37s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.77s/it] +[INFO|modeling_utils.py:3029] 2023-11-15 11:19:35,344 >> All model checkpoint weights were used when initia Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.38s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.74s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,469 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.57s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.99s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,699 >> Using pad_token, but it is not set yet. +ion_config.json +[INFO|configuration_utils.py:575] 2023-11-15 11:19:35,352 >> Generate config GenerationConfig { + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": 2, + "max_length": 4096, + "pad_token_id": 0, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.28.0.dev0" +} + +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,352 >> Using pad_token, but it is not set yet. +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:19:35,352 >> Assigning [PAD] to the pad_token key of the tokenizer +[INFO|tokenization_utils.py:426] 2023-11-15 11:19:35,352 >> Adding [PAD] to the vocabulary + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.32s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.71s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,369 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.28s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.69s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,524 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.33s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.74s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,707 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.34s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.74s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,743 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.30s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.71s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,770 >> Using pad_token, but it is not set yet. +lizing LlamaForCausalLM. + +[INFO|modeling_utils.py:3037] 2023-11-15 11:19:35,760 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf. +If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training. +[INFO|configuration_utils.py:535] 2023-11-15 11:19:35,767 >> loading configuration file /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf/generation_config.json +[INFO|configuration_utils.py:575] 2023-11-15 11:19:35,767 >> Generate config GenerationConfig { + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": 2, + "max_length": 4096, + "pad_token_id": 0, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.28.0.dev0" +} + +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:19:35,767 >> Using pad_token, but it is not set yet. +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:19:35,767 >> Assigning [PAD] to the pad_token key of the tokenizer +[INFO|tokenization_utils.py:426] 2023-11-15 11:19:35,768 >> Adding [PAD] to the vocabulary +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:19:38,365 >> Assigning to the eos_token key of the tokenizer +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:19:38,366 >> Assigning to the bos_token key of the tokenizer +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:19:38,366 >> Assigning to the unk_token key of the tokenizer +Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00000_of_00016.arro[INFO|tokenization_utils_base.py:907] 2023-11-15 11:19:38,760 >> Assigning to the eos_token key of the tokenizer +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:19:38,760 >> Assigning to the bos_token key of the tokenizer +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:19:38,760 >> Assigning to the unk_token key of the tokenizer +Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00000_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00000_of_00016.arrow +Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00001_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00001_of_00016.arrow +Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00002_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00002_of_00016.arrow +Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00003_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00003_of_00016.arrow +Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00004_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00004_of_00016.arrow +Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00005_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00005_of_00016.arrow +Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00006_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00006_of_00016.arrow +Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00007_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00007_of_00016.arrow +Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00008_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00008_of_00016.arrow +Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00009_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00009_of_00016.arrow +Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00010_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00010_of_00016.arrow +Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00011_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00011_of_00016.arrow +Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00012_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00012_of_00016.arrow +Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00013_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00013_of_00016.arrow +Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00014_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00014_of_00016.arrow +Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00015_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00015_of_00016.arrow +Loading cached processed dataset at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_*_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_*_of_00016.arrow +Concatenating 16 shards +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Concatenating 16 shards +Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00000_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00000_of_00016.arrow +Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00001_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00001_of_00016.arrow +Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00002_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00002_of_00016.arrow +Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00003_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00003_of_00016.arrow +Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00004_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00004_of_00016.arrow +Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00005_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00005_of_00016.arrow +Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00006_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00006_of_00016.arrow +Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00007_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00007_of_00016.arrow +Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00008_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00008_of_00016.arrow +Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00009_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00009_of_00016.arrow +Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00010_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00010_of_00016.arrow +Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00011_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00011_of_00016.arrow +Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00012_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00012_of_00016.arrow +Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00013_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00013_of_00016.arrow +Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00014_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00014_of_00016.arrow +Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00015_of_00016.arrow +11/15/2023 11:19:38 - INFO - datasets.arrow_dataset - Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00015_of_00016.arrow +Loading cached processed dataset at /apdcephfs/share_7ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:72674 [2] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:72675 [3] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:72674 [2] NCCL INFO Bootstrap : Using eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:72675 [3] NCCL INFO Bootstrap : Using eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:72678 [6] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:72672 [0] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:72678 [6] NCCL INFO Bootstrap : Using eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:72672 [0] NCCL INFO Bootstrap : Using eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:72674 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:72675 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:72679 [7] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:72677 [5] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:72679 [7] NCCL INFO Bootstrap : Using eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:72677 [5] NCCL INFO Bootstrap : Using eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:72676 [4] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:72673 [1] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:72676 [4] NCCL INFO Bootstrap : Using eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:72673 [1] NCCL INFO Bootstrap : Using eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:72678 [6] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:72672 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:72677 [5] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:72679 [7] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:72673 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:72676 [4] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.64.140<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO Using network IB +45<0> +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:160676 [4] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:160676 [4] NCCL INFO Bootstrap : Using eth1:11.219.11.45<0> +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:160676 [4] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.219.11.45<0> +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO Using network IB +11/15/2023 11:19:47 - WARNING - __main__ - Process rank: 3, device: cuda:3, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:19:47 - WARNING - __main__ - Process rank: 7, device: cuda:7, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:19:47 - WARNING - __main__ - Process rank: 6, device: cuda:6, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:19:47 - WARNING - __main__ - Process rank: 5, device: cuda:5, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:19:47 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:19:47 - WARNING - __main__ - Process rank: 2, device: cuda:2, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:19:47 - WARNING - __main__ - Process rank: 1, device: cuda:1, n_gpu: 1distributed training: True, 16-bits training: False +11/15/2023 11:19:47 - INFO - __main__ - Training/evaluation parameters TrainingArguments( +_n_gpu=1, +adafactor=False, +adam_beta1=0.9, +adam_beta2=0.999, +adam_epsilon=1e-08, +auto_find_batch_size=False, +bf16=True, +bf16_full_eval=True, +data_seed=None, +dataloader_drop_last=False, +dataloader_num_workers=8, +dataloader_pin_memory=True, +ddp_bucket_cap_mb=None, +ddp_find_unused_parameters=None, +ddp_timeout=72000, +debug=[], +deepspeed=train/deepspeed_config_zero2.json, +disable_tqdm=False, +do_eval=False, +do_predict=False, +do_train=True, +eval_accumulation_steps=None, +eval_delay=0, +eval_steps=None, +evaluation_strategy=no, +fp16=False, +fp16_backend=auto, +fp16_full_eval=False, +fp16_opt_level=O1, +fsdp=[], +fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, +fsdp_min_num_params=0, +fsdp_transformer_layer_cls_to_wrap=None, +full_determinism=False, +gradient_accumulation_steps=1, +gradient_checkpointing=True, +greater_is_better=None, +group_by_length=False, +half_precision_backend=auto, +hub_model_id=None, +hub_private_repo=False, +hub_strategy=every_save, +hub_token=, +ignore_data_skip=False, +include_inputs_for_metrics=False, +jit_mode_eval=False, +label_names=None, +label_smoothing_factor=0.0, +learning_rate=2e-05, +length_column_name=length, +load_best_model_at_end=False, +local_rank=0, +log_level=passive, +log_level_replica=warning, +log_on_each_node=True, +logging_dir=/apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20/runs/Nov15_11-18-23_ts-cbba87c5e7504a249f5127103d9ce40f-worker-0, +logging_first_step=False, +logging_nan_inf_filter=True, +logging_steps=10, +logging_strategy=steps, +lr_scheduler_type=cosine, +max_grad_norm=1.0, +max_steps=-1, +metric_for_best_model=None, +mp_parameters=, +no_cuda=False, +num_train_epochs=3.0, +optim=adamw_hf, +optim_args=None, +output_dir=/apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20, +overwrite_output_dir=True, +past_index=-1, +per_device_eval_batch_size=2, +per_device_train_batch_size=48, +prediction_loss_only=False, +push_to_hub=False, +push_to_hub_model_id=None, +push_to_hub_organization=None, +push_to_hub_token=, +ray_scope=last, +remove_unused_columns=True, +report_to=['tensorboard'], +resume_from_checkpoint=None, +run_name=/apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20, +save_on_each_node=False, +save_steps=500, +save_strategy=steps, +save_total_limit=2, +seed=34, +sharded_ddp=[], +skip_memory_metrics=True, +tf32=None, +torch_compile=False, +torch_compile_backend=None, +torch_compile_mode=None, +torchdynamo=None, +tpu_metrics_debug=False, +tpu_num_cores=None, +use_ipex=False, +use_legacy_prediction_loop=False, +use_mps_device=False, +warmup_ratio=0.03, +warmup_steps=0, +weight_decay=0.0, +xpu_backend=None, +) +11/15/2023 11:19:47 - INFO - __main__ - Loading dataset from file: {'train': '/apdcephfs/share_733425/vinnylywang/jianhuipang/datasets/wmt23/wmt23-deen/train.catalpacagpt4.newstests17to20.10000.de2en.hf.json', 'validation': '/apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/test/WMT22/newstest22.de2en.hf.json'} +11/15/2023 11:19:47 - WARNING - __main__ - Process rank: 4, device: cuda:4, n_gpu: 1distributed training: True, 16-bits training: False +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +/apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/load.py:2089: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. +You can remove this warning by passing 'token=None' instead. + warnings.warn( +Using custom data configuration default-b62333a3cd00a31b +11/15/2023 11:19:48 - INFO - datasets.builder - Using custom data configuration default-b62333a3cd00a31b +Loading Dataset Infos from /apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/packaged_modules/json +11/15/2023 11:19:48 - INFO - datasets.info - Loading Dataset Infos from /apdcephfs/share_733425/vinnylywang/jianhuipang/llama2_sft/envs/lib/python3.8/site-packages/datasets/packaged_modules/json +Overwrite dataset info from restored data version if exists. +11/15/2023 11:19:48 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists. +Loading Dataset info from /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96 +11/15/2023 11:19:48 - INFO - datasets.info - Loading Dataset info from /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96 +Found cached dataset json (/apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +11/15/2023 11:19:48 - INFO - datasets.builder - Found cached dataset json (/apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Loading Dataset info from /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96 +11/15/2023 11:19:48 - INFO - datasets.info - Loading Dataset info from /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96 +[INFO|configuration_utils.py:666] 2023-11-15 11:19:48,466 >> loading configuration file /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf/config.json +[INFO|configuration_utils.py:720] 2023-11-15 11:19:48,467 >> Model config LlamaConfig { + "_name_or_path": "/apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf", + "architectures": [ + "LlamaForCausalLM" + ], + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 11008, + "max_position_embeddings": 4096, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 32, + "pad_token_id": 0, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.28.0.dev0", + "use_cache": true, + "vocab_size": 32000 +} + +11/15/2023 11:19:48 - INFO - __main__ - Tokenizer_kwargs: {'cache_dir': None, 'use_fast': True, 'revision': 'main', 'use_auth_token': None} +[INFO|tokenization_utils_base.py:1801] 2023-11-15 11:19:48,477 >> loading file tokenizer.model +[INFO|tokenization_utils_base.py:1801] 2023-11-15 11:19:48,478 >> loading file added_tokens.json +[INFO|tokenization_utils_base.py:1801] 2023-11-15 11:19:48,478 >> loading file special_tokens_map.json +[INFO|tokenization_utils_base.py:1801] 2023-11-15 11:19:48,478 >> loading file tokenizer_config.json +11/15/2023 11:19:48 - INFO - __main__ - Loading checkpoints in dtype: None +[INFO|modeling_utils.py:2395] 2023-11-15 11:19:48,500 >> loading weights file /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf/pytorch_model.bin.index.json +[INFO|configuration_utils.py:575] 2023-11-15 11:19:48,501 >> Generate config GenerationConfig { + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "transformers_version": "4.28.0.dev0" +} + + Loading checkpoint shards: 0%| | 0/2 [00:00> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:08<00:00, 4.12s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:08<00:00, 4.47s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:20:35,278 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.21s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.57s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:20:35,690 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.33s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.71s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:20:35,901 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:10<00:00, 4.66s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:10<00:00, 5.06s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:20:36,375 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:10<00:00, 4.68s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:10<00:00, 5.08s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:20:36,420 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.57s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00, 4.99s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:20:36,463 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [00:10<00:00, 4.69s/it] Loading checkpoint shards: 100%|██████████| 2/2 [00:10<00:00, 5.11s/it] +[INFO|modeling_utils.py:3029] 2023-11-15 11:20:36,727 >> All model checkpoint weights were used when initializing LlamaForCausalLM. + +[INFO|modeling_utils.py:3037] 2023-11-15 11:20:36,727 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf. +If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training. +[INFO|configuration_utils.py:535] 2023-11-15 11:20:36,735 >> loading configuration file /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf/generation_config.json +[INFO|configuration_utils.py:575] 2023-11-15 11:20:36,735 >> Generate config GenerationConfig { + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": 2, + "max_length": 4096, + "pad_token_id": 0, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.28.0.dev0" +} + +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:20:36,735 >> Using pad_token, but it is not set yet. +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:20:36,735 >> Assigning [PAD] to the pad_token key of the tokenizer +[INFO|tokenization_utils.py:426] 2023-11-15 11:20:36,735 >> Adding [PAD] to the vocabulary +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:158760 [2] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:158762 [4] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:158760 [2] NCCL INFO Bootstrap : Using eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:158762 [4] NCCL INFO Bootstrap : Using eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:158760 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:158762 [4] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159743 [2] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159743 [2] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159744 [4] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159744 [4] NCCL INFO Using network IB +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:20:39,743 >> Assigning to the eos_token key of the tokenizer +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:20:39,743 >> Assigning to the bos_token key of the tokenizer +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:20:39,743 >> Assigning to the unk_token key of the tokenizer +Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00000_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00000_of_00016.arrow +Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00001_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00001_of_00016.arrow +Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00002_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00002_of_00016.arrow +Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00003_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00003_of_00016.arrow +Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00004_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00004_of_00016.arrow +Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00005_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00005_of_00016.arrow +Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00006_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00006_of_00016.arrow +Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00007_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00007_of_00016.arrow +Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00008_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00008_of_00016.arrow +Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00009_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00009_of_00016.arrow +Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00010_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00010_of_00016.arrow +Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00011_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00011_of_00016.arrow +Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00012_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00012_of_00016.arrow +Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00013_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00013_of_00016.arrow +Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00014_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00014_of_00016.arrow +Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00015_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_00015_of_00016.arrow +Loading cached processed dataset at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_*_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-f4c7173992cfa371_*_of_00016.arrow +Concatenating 16 shards +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Concatenating 16 shards +Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00000_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00000_of_00016.arrow +Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00001_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00001_of_00016.arrow +Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00002_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00002_of_00016.arrow +Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00003_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00003_of_00016.arrow +Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00004_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00004_of_00016.arrow +Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00005_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00005_of_00016.arrow +Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00006_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00006_of_00016.arrow +Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00007_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00007_of_00016.arrow +Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00008_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00008_of_00016.arrow +Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00009_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00009_of_00016.arrow +Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00010_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00010_of_00016.arrow +Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00011_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00011_of_00016.arrow +Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00012_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00012_of_00016.arrow +Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00013_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00013_of_00016.arrow +Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00014_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00014_of_00016.arrow +Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00015_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00015_of_00016.arrow +Loading cached processed dataset at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_*_of_00016.arrow +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_*_of_00016.arrow +Concatenating 16 shards +11/15/2023 11:20:39 - INFO - datasets.arrow_dataset - Concatenating 16 shards +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158759:158759 [1] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158759:158759 [1] NCCL INFO Bootstrap : Using eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:158761 [3] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:158761 [3] NCCL INFO Bootstrap : Using eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158759:158759 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:158761 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158759:159748 [1] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158759:159748 [1] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159749 [3] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159749 [3] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158763:158763 [5] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158763:158763 [5] NCCL INFO Bootstrap : Using eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158763:158763 [5] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158764:158764 [6] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158764:158764 [6] NCCL INFO Bootstrap : Using eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158764:158764 [6] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158763:159752 [5] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158763:159752 [5] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158765:158765 [7] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158765:158765 [7] NCCL INFO Bootstrap : Using eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158764:159753 [6] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158764:159753 [6] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158765:158765 [7] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158765:159756 [7] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158765:159756 [7] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:158758 [0] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:158758 [0] NCCL INFO Bootstrap : Using eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:158758 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:159758 [0] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.218.9.169<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:159758 [0] NCCL INFO Using network IB + Loading checkpoint shards: 50%|█████ | 1/2 [00:48<00:48, 48.12s/it] Loading checkpoint shards: 50%|█████ | 1/2 [00:48<00:48, 48.86s/it] Loading checkpoint shards: 50%|█████ | 1/2 [00:48<00:48, 48.94s/it] Loading checkpoint shards: 50%|█████ | 1/2 [00:49<00:49, 49.51s/it] Loading checkpoint shards: 50%|█████ | 1/2 [00:49<00:49, 49.62s/it] Loading checkpoint shards: 50%|█████ | 1/2 [00:49<00:49, 49.61s/it] Loading checkpoint shards: 50%|█████ | 1/2 [00:48<00:48, 48.04s/it] Loading checkpoint shards: 50%|█████ | 1/2 [00:49<00:49, 49.89s/it] Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 30.26s/it] Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 33.15s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:21:32,552 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 30.49s/it] Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 33.24s/it] + Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 30.51s/it] Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 33.27s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:21:32,740 >> Using pad_token, but it is not set yet. +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:21:32,747 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 30.44s/it] Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 33.32s/it] + Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 30.47s/it] Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 33.38s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:21:32,920 >> Using pad_token, but it is not set yet. +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:21:32,923 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 30.77s/it] Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 33.37s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:21:32,953 >> Using pad_token, but it is not set yet. + Loading checkpoint shards: 100%|██████████| 2/2 [01:04<00:00, 29.75s/it] Loading checkpoint shards: 100%|██████████| 2/2 [01:04<00:00, 32.49s/it] +[INFO|modeling_utils.py:3029] 2023-11-15 11:21:32,959 >> All model checkpoint weights were used when initializing LlamaForCausalLM. + +[INFO|modeling_utils.py:3037] 2023-11-15 11:21:32,960 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf. +If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training. +[INFO|configuration_utils.py:535] 2023-11-15 11:21:32,967 >> loading configuration file /apdcephfs/share_733425/vinnylywang/jianhuipang/opensourcellms/llama2/Llama-2-7b-hf/generation_config.json +[INFO|configuration_utils.py:575] 2023-11-15 11:21:32,968 >> Generate config GenerationConfig { + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": 2, + "max_length": 4096, + "pad_token_id": 0, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.28.0.dev0" +} + +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:21:32,968 >> Using pad_token, but it is not set yet. +[INFO|tokenization_utils_base.py:907] 2023-11-15 11:21:32,968 >> Assigning [PAD] to the pad_token key of the tokenizer +[INFO|tokenization_utils.py:426] 2023-11-15 11:21:32,968 >> Adding [PAD] to the vocabulary + Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 30.46s/it] Loading checkpoint shards: 100%|██████████| 2/2 [01:06<00:00, 33.33s/it] +[ERROR|tokenization_utils_base.py:1042] 2023-11-15 11:21:33,012 >> Using pad_token, but it is not set yet. +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Setting affinity for GPU 0 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO Setting affinity for GPU 7 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO Setting affinity for GPU 5 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO Setting affinity for GPU 2 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO Setting affinity for GPU 6 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO Setting affinity for GPU 4 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161705 [3] NCCL INFO Setting affinity for GPU 3 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO Trees [0] 3/18/-1->2->-1 [1] 3/-1/-1->2->10 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Channel 00/02 : 0 3 10 15 14 13 12 9 8 11 18 23 22 21 20 17 16 19 26 31 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Channel 01/02 : 0 3 10 15 14 13 12 9 8 11 18 23 22 21 20 17 16 19 26 31 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Trees [0] 1/-1/-1->0->7 [1] 1/-1/-1->0->7 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161705 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] -1/-1/-1->1->0 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO Trees [0] 0/-1/-1->7->6 [1] 0/-1/-1->7->6 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO Channel 00/0 : 2[4b000] -> 7[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Channel 00/0 : 0[e000] -> 3[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO Channel 00/0 : 4[93000] -> 1[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO Channel 01/0 : 2[4b000] -> 7[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Channel 01/0 : 0[e000] -> 3[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO Channel 01/0 : 4[93000] -> 1[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO Channel 00/0 : 5[99000] -> 4[93000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO Channel 00/0 : 6[cb000] -> 5[99000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO Channel 01/0 : 5[99000] -> 4[93000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO Channel 01/0 : 6[cb000] -> 5[99000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161705 [3] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO Channel 00/0 : 7[d0000] -> 6[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO Channel 00/0 : 1[13000] -> 0[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO Channel 01/0 : 7[d0000] -> 6[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO Channel 01/0 : 1[13000] -> 0[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Channel 00/0 : 0[e000] -> 1[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Channel 01/0 : 0[e000] -> 1[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO Channel 00/0 : 4[93000] -> 5[99000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO Channel 01/0 : 4[93000] -> 5[99000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO Channel 00/0 : 6[cb000] -> 7[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Channel 00/0 : 0[e000] -> 7[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO Channel 00/0 : 5[99000] -> 6[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO Channel 01/0 : 6[cb000] -> 7[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Channel 01/0 : 0[e000] -> 7[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO Channel 01/0 : 5[99000] -> 6[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO Channel 00/0 : 7[d0000] -> 0[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO Channel 01/0 : 7[d0000] -> 0[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161705 [3] NCCL INFO Channel 00/0 : 3[51000] -> 10[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO Channel 00/0 : 27[51000] -> 2[4b000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161705 [3] NCCL INFO Channel 01/0 : 3[51000] -> 10[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO Channel 01/0 : 27[51000] -> 2[4b000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161927 [2] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161927 [2] NCCL INFO NCCL_IB_SL set by environment to 3. +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161928 [3] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +2c7a28b697434bc96/cache-b881e5580e9356fd_00000_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #0 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00000_of_00016.arrow +Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00001_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #1 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00001_of_00016.arrow +Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00002_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #2 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00002_of_00016.arrow +Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00003_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #3 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00003_of_00016.arrow +Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00004_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #4 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00004_of_00016.arrow +Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00005_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #5 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00005_of_00016.arrow +Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00006_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #6 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00006_of_00016.arrow +Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00007_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #7 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00007_of_00016.arrow +Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00008_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #8 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00008_of_00016.arrow +Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00009_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #9 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00009_of_00016.arrow +Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00010_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #10 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00010_of_00016.arrow +Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00011_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #11 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00011_of_00016.arrow +Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00012_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #12 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00012_of_00016.arrow +Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00013_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #13 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00013_of_00016.arrow +Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00014_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #14 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00014_of_00016.arrow +Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00015_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Process #15 will write at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_00015_of_00016.arrow +Loading cached processed dataset at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_*_of_00016.arrow +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /apdcephfs/share_733425/vinnylywang/jianhuipang/hf_cache/datasets/json/default-b62333a3cd00a31b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b881e5580e9356fd_*_of_00016.arrow +Concatenating 16 shards +11/15/2023 11:21:36 - INFO - datasets.arrow_dataset - Concatenating 16 shards +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139476:139476 [1] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139476:139476 [1] NCCL INFO Bootstrap : Using eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139476:139476 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139476:140514 [1] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139476:140514 [1] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139481:139481 [6] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139481:139481 [6] NCCL INFO Bootstrap : Using eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139481:139481 [6] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139478:139478 [3] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139478:139478 [3] NCCL INFO Bootstrap : Using eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:139477 [2] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:139477 [2] NCCL INFO Bootstrap : Using eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139478:139478 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:139477 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139481:140516 [6] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139481:140516 [6] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139478:140517 [3] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139478:140517 [3] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:140518 [2] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:140518 [2] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139482:139482 [7] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139482:139482 [7] NCCL INFO Bootstrap : Using eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139479:139479 [4] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139479:139479 [4] NCCL INFO Bootstrap : Using eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139482:139482 [7] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139479:139479 [4] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139479:140523 [4] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139479:140523 [4] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139482:140522 [7] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139482:140522 [7] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139475:139475 [0] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139475:139475 [0] NCCL INFO Bootstrap : Using eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139475:139475 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139475:140526 [0] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139475:140526 [0] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139480:139480 [5] NCCL INFO cudaDriverVersion 11070 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139480:139480 [5] NCCL INFO Bootstrap : Using eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139480:139480 [5] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139480:140528 [5] NCCL INFO NET/IB : Using [0]mlx5_2:1/RoCE [RO]; OOB eth1:11.215.57.242<0> +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139480:140528 [5] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO Setting affinity for GPU 7 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO Setting affinity for GPU 5 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Setting affinity for GPU 2 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO Setting affinity for GPU 6 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO Setting affinity for GPU 3 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Setting affinity for GPU 0 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO Setting affinity for GPU 4 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158764:159753 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139476:140514 [1] NCCL INFO Trees [0] -1/-1/-1->25->24 [1] -1/-1/-1->25->24 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139482:140522 [7] NCCL INFO Trees [0] 24/-1/-1->31->30 [1] 24/-1/-1->3ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159743 [2] NCCL INFO Trees [0] 11/-1/-1->10->19 [1] 11/2/-1->10->26 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:159758 [0] NCCL INFO Trees [0] 9/-1/-1->8->15 [1] 9/-1/-1->8->15 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159744 [4] NCCL INFO Trees [0] 13/-1/-1->12->11 [1] 13/-1/-1->12->11 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159749 [3] NCCL INFO Trees [0] 12/-1/-1->11->10 [1] 12/18/-1->11->10 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158763:159752 [5] NCCL INFO Trees [0] 14/-1/-1->13->12ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO Channel 00/0 : 20[93000] -> 17[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Channel 00/0 : 18[4b000] -> 23[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Channel 00/0 : 16[e000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO Channel 01/0 : 20[93000] -> 17[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Channel 01/0 : 18[4b000] -> 23[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Channel 01/0 : 16[e000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO Channel 00/0 : 21[99000] -> 20[93000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO Channel 00/0 : 22[cb000] -> 21[99000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO Channel 01/0 : 22[cb000] -> 21[99000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO Channel 01/0 : 21[99000] -> 20[93000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO Channel 00/0 : 17[13000] -> 16[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO Channel 00/0 : 23[d0000] -> 22[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO Channel 01/0 : 17[13000] -> 16[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO Channel 01/0 : 23[d0000] -> 22[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Channel 00/0 : 16[e000] -> 17[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Channel 01/0 : 16[e000] -> 17[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO Channel 00/0 : 20[93000] -> 21[99000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO Channel 01/0 : 20[93000] -> 21[99000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Channel 00/0 : 16[e000] -> 23[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO Channel 00/0 : 21[99000] -> 22[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO Channel 00/0 : 22[cb000] -> 23[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Channel 01/0 : 16[e000] -> 23[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO Channel 01/0 : 21[99000] -> 22[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO Channel 01/0 : 22[cb000] -> 23[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO Channel 00/0 : 23[d0000] -> 16[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO Channel 01/0 : 23[d0000] -> 16[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Channel 00/0 : 11[51000] -> 18[4b000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO Channel 00/0 : 19[51000] -> 26[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73634 [1] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73628 [6] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73631 [5] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO Channel 01/0 : 19[51000] -> 26[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Channel 01/0 : 11[51000] -> 18[4b000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73629 [0] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73748 [3] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73632 [7] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73746 [2] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73746 [2] NCCL INFO NCCL_IB_SL set by environment to 3. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO Channel 00/0 : 19[51000] -> 20[93000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Channel 00/0 : 18[4b000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO Channel 01/0 : 19[51000] -> 20[93000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Channel 01/0 : 18[4b000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO Channel 00/0 : 20[93000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO NCCL_NET_GDR_READ set by environment to 1. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73635 [4] NCCL INFO Channel 01/0 : 20[93000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Channel 00/0 : 18[4b000] -> 26[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73627 [3] NCCL INFO Channel 00/0 : 10[4b000] -> 19[51000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73748 [3] NCCL INFO NCCL_IB_SL set by environment to 3. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73626 [2] NCCL INFO Channel 00/0 : 2[4b000] -> 18[4b000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-workets-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159743 [2] NCCL INFO Channel 01/0 : 26[4b0ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159743 [2] NCCL INFO Channel 01/0 : 1ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:7267ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159743 [2] NCCL INFO Channel 00/0 : 19ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159743 [2] NCCL INFO Channel 01/0 : 10[4b000] -> 2[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159812 [3] NCCL INFO NCCL_IB_SL set by environment to 3. +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159749 [3] NCCL INFO Channel 00/0 : 11[51000] -> 10[4b000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159749 [3] NCCLts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161705 [3] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161705 [3] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161705 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:161703 [2] NCCL INFO comm 0x475d0a20 rank 2 nranks 32 cudaDev 2 busId 4b000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160675:161705 [3] NCCL INFO comm 0x47d6f970 rank 3 nranks 32 cudaDev 3 busId 51000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161707 [1] NCCL INFO comm 0x482f1da0 rank 1 nranks 32 cudaDev 1 busId 13000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161702 [0] NCCL INFO comm 0x464ae680 rank 0 nranks 32 cudaDev 0 busId e000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160677:161706 [5] NCCL INFO comm 0x462adff0 rank 5 nranks 32 cudaDev 5 busId 99000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161719 [6] NCCL INFO comm 0x48056350 rank 6 nranks 32 cudaDev 6 busId cb000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161708 [7] NCCL INFO comm 0x494f4ae0 rank 7 nranks 32 cudaDev 7 busId d0000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161721 [4] NCCL INFO comm 0x47bc26e0 rank 4 nranks 32 cudaDev 4 busId 93000 - Init COMPLETE +11/15/2023 11:21:41 - INFO -11/15/2023 11:21:41 - INFO - __main__ - xxx: Showcase the {'input_ids': [1, 13866, 338{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, {'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 1288{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -1[INFO|trainer.py:606] 2023-11-15 11:21:42,462 >> Using cuda_amp half precision backend +[2023-11-15 11:21:42,470] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed info: version=0.11.1, git-hash=unknown, git-branch=unknown +0, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +[INFO|trainer.py:606] 2023-11-15 11:21:42,437 >> Using cuda_amp half precision backend +, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29892, 3300, 2859, 411, 385, 1881, 393, 8128, 4340, 3030, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29902, 29915, 29881, 11188, 372, 565, 366, 1033, 2198, 278, 4223, 13962, 363, 1438, 25260, 29889, 13, 13, 2277, 29937, 10567, 29901, 13, 29906, 29947, 29899, 20460, 4087, 25911, 297, 3087, 8970, 23809, 2025, 1622, 21831, 9784, 13, 13, 2277, 29937, 13291, 29901, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 29906, 29947, 29899, 12883, 29899, 21648, 6561, 29888, 7460, 16992, 472, 3087, 8970, 23809, 2]} +[INFO|trainer.py:606] 2023-11-15 11:21:42,493 >> Using cuda_amp half precision backend +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73780 [1] NCCL INFO Using network IB +tsts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139479:140573 [4] NCCL INFO Using networkts-cts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158765:159859 [7] NCCL INFO Using netwots-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73783 [0] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73784 [5] NCCL INFO Using network IB +ts-cbba87cts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159862 [4] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO Using network IB +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158759:159864 [1] NCCL INFO Uts-cbba87c5e7504ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139479:140573 [4] NCCL INFO Setting affinity for GPU 4 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139482:140575 [7] NCCL INFO Setting affinity for GPU 7 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139481:140578 [6] NCCL INFO Setting affinity for GPU 6 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139480:140576 [5] NCCL INFO Setting affinity for GPU 5 to ffffffff,ffff0000,00000000,ffffffff,ffff0000,00000000 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139476:140574 [1] NCCL INFO Setting affinity for GPU 1 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139478:140577 [3] NCCL INFO Setting affinity for GPU 3 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139475:140579 [0] NCCL INFO Setting affinity for GPU 0 to ffff,ffffffff,00000000,0000ffff,ffffffff +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:140572 [2] NCCL INFO Setting affinity for GPU 2 to ffff,ffffts-cbba87c5e7504a249f5127103d9cets-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:159858 [0] NCCL INFO Trees [0] 9/-1ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73786 [7] NCCL INFO Trees [0] 16ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160679:161973 [7] NCCL ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158763:159857 [5] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->12 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158759:159864 [1] NCCL INFO Trees [0] -1/-1/-1->9->8 [1] -1/-1/-1->9->8 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159862 [4] NCCL INFO Trees [0] 13/-1/-1->12->11 [1] 13/-1/-1->12->11 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158765:159859 [7] NCCL INFO Trees [0] 8/-1/-1->15->14 [1] 8/-1/-1->15->14 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159861 [3] NCCL INFO Trees [0] 12/-1/-1->11->10 [1] 12/18/-1->11->10 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO Trees [0] 11/-1/-1->10->19 [1] 11/2/-1->10->26 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159862 [4] NCCL INFO Channel 00/0 : 12[93000] -> 9[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:159858 [0] NCCL INFO Channel 00/0 : 8[e000] -> 11[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO Channel 00/0 ts-cbba87c5e7504a249f5127103d9ce40f-workerts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159862 [4] NCCL INFO Channel 01/ts-cbba87c5e7504a249f5127103d9ce40f-worker-ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:159858 [0] NCCL INFO Channel 01/0 : 8[e000] -> 11[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO Channel 0ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:7ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158763:159857 [5] NCCL INFO Channelts-cbba87c5e7504a249f5127103d9ce40f-worker-1:726ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158764:159860 [6] NCCL INFO Channts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158764:159860 [6] NCCL INFO Chats-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:7ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158763:159857 [5] NCCL INFO Cts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:737ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158759:159864 [1] NCCL INFO Channel 00/0 : 9[13000] -> 8[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158765:159859 [7] NCCL INFOts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73780 [1] NCCL INFO Channel 01/0 : 17[13000] -> 16[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73786 [7] NCCL INFO Channel 01/0 : 23[d0000] -> 22[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73781 [ts-cbba87c5e7504a249f5127103d9ce4ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73786 [7]ts-cbba87c5e7504a249f5127103d9cts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73783 [0] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73783 [0] NCCL INFO Channel 00/0 : 16[e000] -> 17[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73780 [1] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73785 [6] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73783 [0] NCCL INFO Channel 01/0 : 16[e000] -> 17[13000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73784 [5] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73781 [4] NCCL INFO Channel 00/0 : 20[9300ts-cbba87c5e7504a249f5127103d9ce4ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73781 [4] NCCL INFO Cts-cbba87c5e7504a249f51ts-cbba87c5e7504a249f5127103d9cts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73785 [6] NCCL INFO Channel 00/0 : 22[cb000] -ts-cbba87c5e7504a249f5127103dts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73784 [5] NCCL INFO Channel 00/0 : 21[99000] ->ts-cbbats-cbba87c5e7504a249f5127103d9ce40f-launcher:160676:161976 [4] NCCL INFO Channel 01/0 : 4[93000] -> 5[99000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161975 [1] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161975 [1] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160673:161975 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e75ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73785 [6] NCCL INFO Channel 01/0 : 22[cb000] -> 23[d0000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72677:73784 [5] NCCL INFO Channel 01/0 : 21[99000] -> 22[cb000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73783 [0] NCCL INFO Channel 01/0 : 16[e000] -> 23[d0000] viats-cbba87c5e75ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73785 [6] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73785 [6] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73785 [6] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f51271ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161978 [6] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161978 [6] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160678:161978 [6] NCCL INFO 2 coll channels, 2 p2p chts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73786 [7] NCCL INFO Channel 00/0 : 23[d0000] -> 16[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73786 [7] NCCL INFO Channel 01/0 : 23[d0000] -> 16[e000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73783 [0] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73783 [0] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73783 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73786 [7] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73786 [7] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72679:73786 [7] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161971 [0] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161971 [0] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a2ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73782 [3] NCCL INFO Channel 01/0 : 19[51000] -> 26[4b000] [send] viats-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO Channel 01/0 : 3[51000] -> 10[4b000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73782 [3] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73782 [3] NCCL INFO Channel 00/0 : 19[51000] -> 20[93000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73782 [3] NCCL INFO Channel 01/0 : 19[51000] -> 20[93000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73787 [2] NCCL INFO Connected all rings +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73787 [2] NCCL INFO Channel 00/0 : 18[4b000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73787 [2] NCCL INFO Channel 01/0 : 18[4b000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73781 [4] NCCL INFO Channel 00/0 : 20[93000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73781 [4] NCCL INFO Channel 01/0 : 20[93000] -> 19[51000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73787 [2] NCCL INFO Channel 00/0 : 18[4b000] -> 26[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73782 [3] NCCL INFO Channel 00/0 : 10[4b000] -> 19[51000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73787 [2] NCCL INFO Channel 00/0 : 2[4b000] -> 18[4b000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73787 [2] NCCL INFO Channel 00/0 : 18[4b000] -> 2[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73782 [3] NCCL INFO Channel 00/0 : 19[51000] -> 10[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73787 [2] NCCL INFO Channel 00/0 : 26[4b000] -> 18[4b000] [receive] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:73782 [3] NCCL INFO Channel 00/0 : 19[51000] -> 18[4b000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO Channel 01/0 : 10[4b000] -> 2[4b000] [send] via NET/IB/0/GDRDMA +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159861 [3] NCCL INFO Channel 00/0 : 11[51000] -> 10[4b000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159861 [3] NCCL INFO Channel 01/0 : 11[51000] -> 10[4b000] via P2P/IPC/read +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159862 [4] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159862 [4] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158762:159862 [4] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO threadThresholds 8/8/64 | 256/8/64 | 512 | 512 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159861 [3] NCCL INFO Connected all trees +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159861 [3] NCCL INFO ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72673:73780 [1] NCCL INFO comm 0x78ef680 rank 17 nranks 32 cudaDev 1 busId 13000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40fts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:159863 [2] NCCL INFO comm 0xad90340 rank 10 nranks 32 cudaDev 2 busId 4b000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158761:159861 [3] NCCL INFO comm 0x75f3f00 rank 11 nranks 32 cudaDev 3 busId 51000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158759:159864 [1] NCCL INFO comm 0x8348640 rank 9 nranks 32 cudaDev 1 busId 13000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158758:159858 [0] NCCL INFO comm 0x967a800 rank 8 nranks 32 cudaDev 0 busId e000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158764:159860 [6] NCCL INFO comm 0x9d261c0 rank 14 nranks 32 cudaDev 6 busId cb000 - Init COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139478:140588 [3] NCCL INFO NCCL_IB_SL set by environment to 3. +Using /root/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... +Using /root/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... +Using /root/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... +Detected CUDA files, patching ldflags +Emitting ninja build file /root/.cache/torch_extensions/py38_cu117/cpu_adam/build.ninja... +Building extension module cpu_adam... +Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +ninja: no work to do. +Using /root/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 1.0798652172088623 seconds +Loading extension module cpu_adam... +Time to load cpu_adam op: 0.9681258201599121 seconds +Loading extension module cpu_adam... +Time to load cpu_adam op: 1.1238970756530762 seconds +Loading extension module cpu_adam... +Time to load cpu_adam op: 0.8728253841400146 seconds +Using /root/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... +Using /root/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... +Using /root/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... +Using /root/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... +Detected CUDA files, patching ldflags +Emitting ninja build file /root/.cache/torch_extensions/py38_cu117/cpu_adam/build.ninja... +Building extension module cpu_adam... +Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +ninja: no work to do. +Loading extension module cpu_adam... +Time to load cpu_adam op: 1.1005303859710693 seconds +Loading extension module cpu_adam... +Time to load cpu_adam op: 1.0993423461914062 seconds +Loading extension module cpu_adam... +Time to load cpu_adam op: 1.152484655380249 seconds +Loading extension module cpu_adam... +Time to load cpu_adam op: 1.1483979225158691 seconds +Adam Optimizer #0 is created with AVX2 arithmetic capability. +Config: alpha=0.000020, betas=(0.900000, 0.999000), weight_decay=0.000000, adam_w=1 +OBS=N) +Using /root/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... +Using /root/.cache/torch_extensionsLoading extension module cpu_adam... +Time ninja: no workAdam Optimizer #0 is created with AVX2 arithmetic capability. +Config: alpha=0.000020, betas=(0.900000, 0.999000), weight_decay=0.000000, adam_w=1 +[2023-11-15 11:22:02,804] [INFO] [logging.py:96:log_dist] [Rank 0] Using DeepSpeed Optimizer param name adam as basic optimizer +[2023-11-15 11:22:02,804] [INFO] [logging.py:96:log_dist] [Rank 0] Removing param_group that has no 'params' in the basic Optimizer +[2023-11-15 11:22:02,823] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Basic Optimizer = DeepSpeedCPUAdam +[2023-11-15 11:22:02,823] [INFO] [utils.py:56:is_zero_supported_optimizer] Checking ZeRO support for optimizer=DeepSpAdam Optimizer #0 is created with AVX2 arithmetic capability. +Config: alpha=0.000020, betas=(0.900000, 0.999000), weight_decay=0.000000, adam_w=1 + torch.bfloat16 ZeRO stage 2 optimizer +[2023-11-15 11:22:02,823] [INFO] [stage_1_and_2.py:146:__init__] Reduce bucket size 200000000 +[2023-11-15 11:22:02,823] [INFO] [stage_1_and_2.py:147:__init__] Allgather bucket size 200000000 +[2023-11-15 11:22:02,823] [INFO] [stage_1_and_2.py:148:__init__] CPU Offload: True +[2023-11-15 11:22:02,824] [INFO] [stage_1_and_2.py:149:__init__] Round robin gradient partitioning: False +[INFO|trainer.py:1755] 2023-11-15 11:22:19,884 >> ***** Running training ***** +[INFO|trainer.py:1756] 2023-11-15 11:22:19,884 >> Num examples = 64204 +[INFO|trainer.py:1757] 2023-11-15 11:22:19,884 >> Num Epochs = 3 +[INFO|trainer.py:1758] 2023-11-15 11:22:19,884 >> Instantaneous batch size per device = 48 +[INFO|trainer.py:1759] 2023-11-15 11:22:19,884 >> Total train batch size (w. parallel, distributed & accumulation) = 1536 +[INFO|trainer.py:1760] 2023-11-15 11:22:19,884 >> Gradient Accumulation steps = 1 +[INFO|trainer.py:1761] 2023-11-15 11:22:19,884 >> [WARNING|logging.py:295] 2023-11-15 11:22:19,920 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295][WARNING|logging.py:295] 2023-11-15 11:22:20,117 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295] 2023-11-15 11:22:20,549 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295] 2023-11-15 11:22:20,567 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295] 2023-11-15 11:22:20,686 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295] 2023-11-15 11:22:21,779 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295] 2023-11-15 11:22:21,789 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295] 2023-11-15 11:22:21,816 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295] 2023-11-15 11:22:21,863 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... + CA 12.83 GB Max_CA 13 GB +[2023-11-15 11:22:20,210] [INFO] [utils.py:810:see_memory_usage] CPU Virtual Memory: used = 78.59 GB, percent = 7.8% +[2023-11-15 11:22:20,210] [INFO] [stage_1_and_2.py:513:__init__] optimizer state initialized +[WARNING|logging.py:295] 2023-11-15 11:22:20,216 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[2023-11-15 11:22:20,316] [INFO] [utils.py:802:see_memory_usage] After initializing ZeRO optimizer +[2023-11-15 11:22:20,317] [INFO] [utils.py:803:see_memory_usage] MA 12.83 GB Max_MA 12.83 GB CA 12.83 GB Max_CA 13 GB +[2023-11-15 11:22:20,317] [INFO] [utils.py:810:see_memory_usage] CPU Virtual Memory: used = 78.63 GB, percent = 7.8% +[2023-11-15 11:22:20,318] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Final Optimizer = adam +[2023-11-15 11:22:20,318] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed using client callable to create LR scheduler +[2023-11-15 11:22:20,318] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed LR Scheduler = +[2023-11-15 11:22:20,319] [INFO] [logging.py:96:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0], mom=[[0.9, 0.999]] +[2023-11-15 11:22:20,320] [INFO] [config.py:968:print] DeepSpeedEngine configuration: +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] activation_checkpointing_config { + "partition_activations": false, + "contiguous_memory_optimization": false, + "cpu_checkpointing": false, + "number_checkpoints": null, + "synchronize_checkpoint_boundary": false, + "profile": false +} +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] amp_enabled .................. False +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] amp_params ................... False +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] autotuning_config ............ { + "enabled": false, + "start_step": null, + "end_step": null, + "metric_path": null, + "arg_mappings": null, + "metric": "throughput", + "model_info": null, + "results_dir": "autotuning_results", + "exps_dir": "autotuning_exps", + "overwrite": true, + "fast": true, + "start_profile_step": 3, + "end_profile_step": 5, + "tuner_type": "gridsearch", + "tuner_early_stopping": 5, + "tuner_num_trials": 50, + "model_info_path": null, + "mp_size": 1, + "max_train_batch_size": null, + "min_train_batch_size": 1, + "max_train_micro_batch_size_per_gpu": 1.024000e+03, + "min_train_micro_batch_size_per_gpu": 1, + "num_tuning_micro_batch_sizes": 3 +} +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] bfloat16_enabled ............. True +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] checkpoint_parallel_write_pipeline False +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] checkpoint_tag_validation_enabled True +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] checkpoint_tag_validation_fail False +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] comms_config ................. +[2023-11-15 11:22:20,320] [INFO] [config.py:972:print] communication_data_type ...... None +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] curriculum_enabled_legacy .... False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] curriculum_params_legacy ..... False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}} +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] data_efficiency_enabled ...... False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] dataloader_drop_last ......... False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] disable_allgather ............ False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] dump_state ................... False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] dynamic_loss_scale_args ...... None +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] eigenvalue_enabled ........... False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] eigenvalue_gas_boundary_resolution 1 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] eigenvalue_layer_name ........ bert.encoder.layer +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] eigenvalue_layer_num ......... 0 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] eigenvalue_max_iter .......... 100 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] eigenvalue_stability ......... 1e-06 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] eigenvalue_tol ............... 0.01 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] eigenvalue_verbose ........... False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] elasticity_enabled ........... False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] flops_profiler_config ........ { + "enabled": false, + "recompute_fwd_factor": 0.0, + "profile_step": 1, + "module_depth": -1, + "top_modules": 1, + "detailed": true, + "output_file": null +} +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] fp16_auto_cast ............... None +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] fp16_enabled ................. False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] fp16_master_weights_and_gradients False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] global_rank .................. 0 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] grad_accum_dtype ............. None +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] gradient_accumulation_steps .. 1 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] gradient_clipping ............ 1.0 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] gradient_predivide_factor .... 1.0 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] hybrid_engine ................ enabled=False max_out_tokens=512 inference_tp_size=1 release_inference_cache=False pin_parameters=True tp_gather_partition_size=8 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] initial_dynamic_scale ........ 1 +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] load_universal_checkpoint .... False +[2023-11-15 11:22:20,321] [INFO] [config.py:972:print] loss_scale ................... 1.0 +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] memory_breakdown ............. False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] mics_hierarchial_params_gather False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] mics_shard_size .............. -1 +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] monitor_config ............... tensorboard=TensorBoardConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') wandb=WandbConfig(enabled=False, group=None, team=None, project='deepspeed') csv_monitor=CSVConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') enabled=False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] nebula_config ................ { + "enabled": false, + "persistent_storage_path": null, + "persistent_time_interval": 100, + "num_of_version_in_retention": 2, + "enable_nebula_load": true, + "load_path": null +} +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] optimizer_legacy_fusion ...... False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] optimizer_name ............... adam +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] optimizer_params ............. {'lr': 2e-05, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0.0} +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] pld_enabled .................. False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] pld_params ................... False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] prescale_gradients ........... False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] scheduler_name ............... None +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] scheduler_params ............. None +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] sparse_attention ............. None +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] sparse_gradients_enabled ..... False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] steps_per_print .............. 1000 +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] train_batch_size ............. 1536 +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] train_micro_batch_size_per_gpu 48 +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] use_node_local_storage ....... False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] wall_clock_breakdown ......... False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] weight_quantization_config ... None +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] world_size ................... 32 +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] zero_allow_untested_optimizer False +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] zero_config .................. stage=2 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=200000000 allgather_partitions=True allgather_bucket_size=200000000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='cpu', nvme_path=None, buffer_count=4, pin_memory=True, pipeline=False, pipeline_read=False, pipeline_write=False, fast_init=False) sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50,000,000 param_persistence_threshold=100,000 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False zero_hpz_partition_size=1 zero_quantized_weights=False zero_quantized_nontrainable_weights=False zero_quantized_gradients=False mics_shard_size=-1 mics_hierarchical_params_gather=False memory_efficient_linear=True pipeline_loading_checkpoint=False override_module_apply=True +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] zero_enabled ................. True +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] zero_force_ds_cpu_optimizer .. True +[2023-11-15 11:22:20,322] [INFO] [config.py:972:print] zero_optimization_stage ...... 2 +[2023-11-15 11:22:20,323] [INFO] [config.py:958:print_user_config] json = { + "optimizer": { + "type": "Adam", + "params": { + "lr": 2e-05, + "betas": [0.9, 0.999], + "eps": 1e-08, + "weight_decay": 0.0 + } + }, + "bf16": { + "enabled": true + }, + "fp16": { + "enabled": false, + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "zero_optimization": { + "stage": 2, + "offload_optimizer": { + "device": "cpu", + "pin_memory": true + }, + "allgather_partitions": true, + "allgather_bucket_size": 2.000000e+08, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 2.000000e+08, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": 1, + "gradient_clipping": 1.0, + "steps_per_print": 1000, + "train_batch_size": 1.536000e+03, + "train_micro_batch_size_per_gpu": 48, + "wall_clock_breakdown": false +} +[INFO|trainer.py:1755] 2023-11-15 11:22:20,324 >> ***** Running training ***** +[INFO|trainer.py:1756] 2023-11-15 11:22:20,324 >> Num examples = 64204 +[INFO|trainer.py:1757] 2023-11-15 11:22:20,324 >> Num Epochs = 3 +[INFO|trainer.py:1758] 2023-11-15 11:22:20,324 >> Instantaneous batch size per device = 48 +[INFO|trainer.py:1759] 2023-11-15 11:22:20,324 >> Total train batch size (w. parallel, distributed & accumulation) = 1536 +[INFO|trainer.py:1760] 2023-11-15 11:22:20,324 >> Gradient Accumulation steps = 1 +[INFO|trainer.py:1761] 2023-11-15 11:22:20,324 >> Total optimization steps = 126 +[INFO|trainer.py:1762] 2023-11-15 11:22:20,325 >> Number of trainable parameters = 6738423808 + 0%| | 0/126 [00:00> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295] 2023-11-15 11:22:20,701 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... +[WARNING|logging.py:295] 2023-11-15 11:22:20,971 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`... + 1%| | 1/126 [00:20<41:58, 20.15s/it] 2%|▏ | 2/126 [00:33<34:02, 16.47s/it] 2%|▏ | 3/126 [00:49<33:25, 16.31s/it] 3%|▎ | 4/126 [01:05<32:46, 16.12s/it] 4%|▍ | 5/126 [01:23<32:32, 16.14s/it] 5%|▍ | 6/126 [01:36<31:44, 15.87s/it] 6%|▌ | 7/126 [01:52<31:26, 15.86s/it] 6%|▋ | 8/126 [02:08<31:10, 15.85s/it] 7%|▋ | 9/126 [02:24<30:54, 15.85s/it] 8%|▊ | 10/126 [02:39<30:30, 15.78s/it] {'loss': 0.9998, 'learning_rate': 1.9880878960910772e-05, 'epoch': 0.24} + 8%|▊ | 10/126 [02:39<30:30, 15.78s/it] 9%|▊ | 11/126 [02:55<30:14, 15.78s/it] 10%|▉ | 12/126 [03:13<29:57, 15.76s/it] 10%|█ | 13/126 [03:29<29:40, 15.76s/it] 11%|█ | 14/126 [03:44<29:21, 15.73s/it] 12%|█▏ | 15/126 [04:00<28:58, 15.67s/it] 13%|█▎ | 16/126 [04:16<28:41, 15.65s/it] 13%|█▎ | 17/126 [04:31<28:26, 15.66s/it] 14%|█▍ | 18/126 [04:45<28:15, 15.70s/it] 15%|█▌ | 19/126 [05:01<28:05, 15.75s/it] 16%|█▌ | 20/126 [05:16<27:45, 15.72s/it] {'loss': 0.909, 'learning_rate': 1.916316904487005e-05, 'epoch': 0.48} + 16%|█▌ | 20/126 [05:16<27:45, 15.72s/it] 17%|█▋ | 21/126 [05:34<27:29, 15.71s/it] 17%|█▋ | 22/126 [05:50<27:14, 15.72s/it] 18%|█▊ | 23/126 [06:06<26:59, 15.72s/it] 19%|█▉ | 24/126 [06:19<26:46, 15.75s/it] 20%|█▉ | 25/126 [06:35<26:38, 15.82s/it] 21%|██ | 26/126 [06:51<26:23, 15.83s/it] 21%|██▏ | 27/126 [07:07<26:07, 15.84s/it] 22%|██▏ | 28/126 [07:23<25:50, 15.82s/it] 23%|██▎ | 29/126 [07:38<25:30, 15.77s/it] 24%|██▍ | 30/126 [07:54<25:16, 15.80s/it] {'loss': 0.8932, 'learning_rate': 1.7841198065767107e-05, 'epoch': 0.71} + 24%|██▍ | 30/126 [07:54<25:16, 15.80s/it] 25%|██▍ | 31/126 [08:12<25:00, 15.79s/it] 25%|██▌ | 32/126 [08:26<24:40, 15.75s/it] 26%|██▌ | 33/126 [08:41<24:23, 15.73s/it] 27%|██▋ | 34/126 [08:57<24:05, 15.71s/it] 28%|██▊ | 35/126 [09:13<23:51, 15.73s/it] 29%|██▊ | 36/126 [09:29<23:37, 15.75s/it] 29%|██▉ | 37/126 [09:44<23:21, 15.75s/it] 30%|███ | 38/126 [10:00<23:05, 15.74s/it] 31%|███ | 39/126 [10:16<22:47, 15.71s/it] 32%|███▏ | 40/126 [10:31<22:33, 15.73s/it] {'loss': 0.8899, 'learning_rate': 1.6002142805483686e-05, 'epoch': 0.95} + 32%|███▏ | 40/126 [10:31<22:33, 15.73s/it] 33%|███▎ | 41/126 [10:47<22:15, 15.72s/it] 33%|███▎ | 42/126 [11:01<21:10, 15.13s/it] 34%|███▍ | 43/126 [11:19<22:06, 15.98s/it] 35%|███▍ | 44/126 [11:34<21:39, 15.85s/it] 36%|███▌ | 45/126 [11:50<21:15, 15.75s/it] 37%|███▋ | 46/126 [12:05<20:50, 15.64s/it] 37%|███▋ | 47/126 [12:21<20:29, 15.56s/it] 38%|███▊ | 48/126 [12:36<20:10, 15.52s/it] 39%|███▉ | 49/126 [12:52<19:57, 15.56s/it] 40%|███▉ | 50/126 [13:07<19:40, 15.53s/it] {'loss': 0.8681, 'learning_rate': 1.3767278936351853e-05, 'epoch': 1.19} + 40%|███▉ | 50/126 [13:07<19:40, 15.53s/it] 40%|████ | 51/126 [13:23<19:25, 15.54s/it] 41%|████▏ | 52/126 [13:38<19:09, 15.54s/it] 42%|████▏ | 53/126 [13:54<18:55, 15.55s/it] 43%|████▎ | 54/126 [14:10<18:43, 15.60s/it] 44%|████▎ | 55/126 [14:25<18:31, 15.65s/it] 44%|████▍ | 56/126 [14:41<18:16, 15.66s/it] 45%|████▌ | 57/126 [14:56<17:59, 15.65s/it] 46%|████▌ | 58/126 [15:12<17:42, 15.62s/it] 47%|████▋ | 59/126 [15:28<17:26, 15.62s/it] 48%|████▊ | 60/126 [15:43<17:12, 15.65s/it] {'loss': 0.866, 'learning_rate': 1.1283983551465512e-05, 'epoch': 1.43} + 48%|████▊ | 60/126 [15:43<17:12, 15.65s/it] 48%|████▊ | 61/126 [15:59<16:57, 15.66s/it] 49%|████▉ | 62/126 [16:15<16:41, 15.65s/it] 50%|█████ | 63/126 [16:30<16:25, 15.64s/it] 51%|█████ | 64/126 [16:46<16:09, 15.64s/it] 52%|█████▏ | 65/126 [17:01<15:52, 15.61s/it] 52%|█████▏ | 66/126 [17:17<15:37, 15.62s/it] 53%|█████▎ | 67/126 [17:33<15:22, 15.64s/it] 54%|█████▍ | 68/126 [17:48<15:08, 15.66s/it] 55%|█████▍ | 69/126 [18:04<14:52, 15.65s/it] 56%|█████▌ | 70/126 [18:20<14:35, 15.63s/it] {'loss': 0.8606, 'learning_rate': 8.71601644853449e-06, 'epoch': 1.67} + 56%|█████▌ | 70/126 [18:20<14:35, 15.63s/it] 56%|█████▋ | 71/126 [18:35<14:20, 15.64s/it] 57%|█████▋ | 72/126 [18:53<14:02, 15.60s/it] 58%|█████▊ | 73/126 [19:07<13:48, 15.63s/it] 59%|█████▊ | 74/126 [19:22<13:34, 15.67s/it] 60%|█████▉ | 75/126 [19:38<13:19, 15.68s/it] 60%|██████ | 76/126 [19:54<13:04, 15.68s/it] 61%|██████ | 77/126 [20:09<12:48, 15.68s/it] 62%|██████▏ | 78/126 [20:25<12:32, 15.68s/it] 63%|██████▎ | 79/126 [20:41<12:18, 15.71s/it] 63%|██████▎ | 80/126 [20:57<12:03, 15.73s/it] {'loss': 0.8603, 'learning_rate': 6.232721063648148e-06, 'epoch': 1.9} + 63%|██████▎ | 80/126 [20:57<12:03, 15.73s/it] 64%|██████▍ | 81/126 [21:15<11:47, 15.73s/it] 65%|██████▌ | 82/126 [21:28<11:32, 15.74s/it] 66%|██████▌ | 83/126 [21:47<11:19, 15.79s/it] 67%|██████▋ | 84/126 [21:58<10:36, 15.16s/it] 67%|██████▋ | 85/126 [22:18<11:19, 16.58s/it] 68%|██████▊ | 86/126 [22:33<10:51, 16.28s/it] 69%|██████▉ | 87/126 [22:49<10:25, 16.03s/it] 70%|██████▉ | 88/126 [23:05<10:04, 15.92s/it] 71%|███████ | 89/126 [23:20<09:43, 15.77s/it] 71%|███████▏ | 90/126 [23:35<09:24, 15.67s/it] {'loss': 0.8592, 'learning_rate': 3.997857194516321e-06, 'epoch': 2.14} + 71%|███████▏ | 90/126 [23:35<09:24, 15.67s/it] 72%|███████▏ | 91/126 [23:50<09:05, 15.59s/it] 73%|███████▎ | 92/126 [24:06<08:51, 15.63s/it] 74%|███████▍ | 93/126 [24:22<08:35, 15.62s/it] 75%|███████▍ | 94/126 [24:38<08:18, 15.58s/it] 75%|███████▌ | 95/126 [24:53<08:04, 15.63s/it] 76%|███████▌ | 96/126 [25:09<07:48, 15.61s/it] 77%|███████▋ | 97/126 [25:25<07:32, 15.62s/it] 78%|███████▊ | 98/126 [25:40<07:17, 15.64s/it] 79%|███████▊ | 99/126 [25:58<07:00, 15.58s/it] 79%|███████▉ | 100/126 [26:14<06:44, 15.55s/it] {'loss': 0.8497, 'learning_rate': 2.158801934232897e-06, 'epoch': 2.38} + 79%|███████▉ | 100/126 [26:11<06:44, 15.55s/it] 80%|████████ | 101/126 [26:29<06:29, 15.56s/it] 81%|████████ | 102/126 [26:42<06:13, 15.57s/it] 82%|████████▏ | 103/126 [26:58<05:58, 15.57s/it] 83%|████████▎ | 104/126 [27:16<05:42, 15.58s/it] 83%|████████▎ | 105/126 [27:29<05:27, 15.60s/it] 84%|███��████▍ | 106/126 [27:47<05:12, 15.64s/it] 85%|████████▍ | 107/126 [28:00<04:56, 15.62s/it] 86%|████████▌ | 108/126 [28:18<04:40, 15.60s/it] 87%|████████▋ | 109/126 [28:34<04:24, 15.58s/it] 87%|████████▋ | 110/126 [28:47<04:09, 15.60s/it] {'loss': 0.8514, 'learning_rate': 8.368309551299536e-07, 'epoch': 2.62} + 87%|████████▋ | 110/126 [28:47<04:09, 15.60s/it] 88%|████████▊ | 111/126 [29:02<03:53, 15.59s/it] 89%|████████▉ | 112/126 [29:18<03:37, 15.55s/it] 90%|████████▉ | 113/126 [29:34<03:22, 15.56s/it] 90%|█████████ | 114/126 [29:49<03:06, 15.55s/it] 91%|█████████▏| 115/126 [30:05<02:50, 15.54s/it] 92%|█████████▏| 116/126 [30:20<02:35, 15.55s/it] 93%|█████████▎| 117/126 [30:35<02:19, 15.52s/it] 94%|█████████▎| 118/126 [30:51<02:04, 15.53s/it] 94%|█████████▍| 119/126 [31:07<01:48, 15.54s/it] 95%|█████████▌| 120/126 [31:22<01:33, 15.60s/it] {'loss': 0.8521, 'learning_rate': 1.1912103908922945e-07, 'epoch': 2.86} + 95%|█████████▌| 120/126 [31:22<01:33, 15.60s/it] 96%|█████████▌| 121/126 [31:38<01:17, 15.57s/it] 97%|█████████▋| 122/126 [31:54<01:02, 15.57s/it] 98%|█████████▊| 123/126 [32:10<00:46, 15.61s/it] 98%|█████████▊| 124/126 [32:25<00:31, 15.60s/it] 99%|█████████▉| 125/126 [32:43<00:15, 15.56s/it] 100%|██████████| 126/126 [32:56<00:00, 14.98s/it][INFO|trainer.py:2025] 2023-11-15 11:55:15,490 >> + +Training completed. Do not forget to share your model on huggingface.co/models =) + + + {'train_runtime': 1975.1648, 'train_samples_per_second': 97.517, 'train_steps_per_second': 0.064, 'train_loss': 0.877318367125496, 'epoch': 3.0} + 100%|██████████| 126/126 [32:55<00:00, 14.98s/it] 100%|██████████| 126/126 [32:55<00:00, 15.68s/it] +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139480:140531 [5] NCCL INFO [Service thread] Connection closed by localRank 5 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:140537 [2] NCCL INFO [Service thread] Connection closed by localRank 5 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139480:139480 [5] NCCL INFO comm 0x4648e800 rank 29 nranks 32 cudaDev 5 busId 99000 - Abort COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139476:140534 [1] NCCL INFO [Service thread] Connection closed by localRank 1 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:140537 [2] NCCL INFO [Service thread] Connection closed by localRank 1 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:140537 [2] NCCL INFO [Service thread] Connection closed by localRank 4 +ts-cbba87c5ets-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73746 [2] NCCL INFO [Service thread] Connection closed by localRank 6 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:73747 [6] NCCL INFO [Service thread] Connection closed by localRank 6 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72678:72678 [6] NCCL INFO comm 0x48b89f70 rank 22 nranks 32 cudaDev 6 busId cb000 - Abort COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:73746 [2] NCCL INFO [Service thread] Connection closed by localRank 4 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:73750 [4] NCCL INFO [Service thread] Connection closed by localRank 4 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72676:72676 [4] NCCL INFO comm 0x49abd180 rank 20 nranks 32 cudaDev 4 busId 93000 - Abort COMPLETE +ts-cbba87c5e7504a249f51ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139475:140533 [0] NCCL INFO [Service thread] Connection closed by localRank 0 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:140537 [2] NCCL INFO [Service thread] Connection closed by localRank 0 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:140537 [2] NCCL INFO [Service thread] Connection closed by localRts-cbbts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139475:139475 [0] NCCL INFO comm 0x483445e0 rank 24 nranks 32 cudaDev ts-cbba87c5e7504a249f5127103d9ts-cbba87c5e7504a249f5127103d9ce40f-worker-2:139477:139477 [2] NCCL INFO comm 0x46d6ea80 rats-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:73749 [0] NCCL INFO [Service thread] Connection closed by localRank 0 +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72675:72675 [3] NCCL INFO comm 0x485a0880 rank 19 nranks 32 cudaDev 3 busId 51000 - Abort COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72672:72672 [0] NCCL INFO comm 0x46eb1080 rank 16 nranks 32 cudaDev 0 busId e000 - Abort COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-1:72674:72674 [2] NCCL INFO comm 0x49f2ace0 rank 18 nranks 32 cudaDev 2 busId 4b000 - Abort COMPLETE +2 cudaDev 6 busId cb000 - Abort COMPLETE +761:158761 [3] NCCL INFO comm 0x463fd4c0 rank 11 nranks 32 cudaDev 3 busId 51000 - Abort COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-worker-0:158760:158760 [2] NCCL INFO comm 0x474144b0 rank 10 nranks 32 cudaDev 2 busId 4b000 - Abort COMPLETE +] NCCL INFO comm 0x462adff0 rank 5 nranks 32 cudaDev 5 busId 99000 - Abort COMPLETE +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160674:160674 [2] NCCL INFO comm 0x475d0a20 rank 2 nranks 32 cudaDev 2 busId 4b000 - Abort COMPLETE +[INFO|modeling_utils.py:1767] 2023-11-15 11:55:57,973 >> The model is bigger than the maximum size per checkpoint (10GB) and is going to be split in 2 checkpoint shards. You can find where each parameters has been saved in the index located at /apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20/pytorch_model.bin.index.json. +[INFO|tokenization_utils_base.py:2164] 2023-11-15 11:55:57,978 >> tokenizer config file saved in /apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20/tokenizer_config.json +[INFO|tokenization_utils_base.py:2171] 2023-11-15 11:55:57,981 >> Special tokens file saved in /apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20/special_tokens_map.json +[INFO|tokenization_utils_base.py:2221] 2023-11-15 11:55:57,984 >> added tokens file saved in /apdcephfs/share_733425/vinnylywang/jianhuipang/LLMs4MT/model/newptmodel-llms4mt-de2en-32a100/llama2-sfton-10000-bitexts-and-alpacagpt4-and-newstests17to20/added_tokens.json +***** train metrics ***** + epoch = 3.0 + train_loss = 0.8773 + train_runtime = 0:32:55.16 + train_samples = 64204 + train_samples_per_second = 97.517 + train_steps_per_second = 0.064 +[INFO|modelcard.py:451] 2023-11-15 11:55:58,020 >> Dropping the following result as it does not have all the necessary fields: +{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}} +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:161924 [0] NCCL INFO [Service thread] Connection closed by localRank 0 +ts-cbba87c5e7504a249f5127103d9ce40f-launcher:160672:160672 [0] NCCL INFO comm 0x464ae680 rank 0 nranks 32 cudaDev 0 busId e000 - Abort COMPLETE