diff --git "a/nohup.out" "b/nohup.out" --- "a/nohup.out" +++ "b/nohup.out" @@ -2,12 +2,12 @@ wandb: Currently logged in as: sanchit-gandhi (use `wandb login --relogin` to fo wandb: wandb version 0.12.17 is available! To upgrade, please run: wandb: $ pip install wandb --upgrade wandb: Tracking run with wandb version 0.12.15 -wandb: Run data is saved locally in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-feature-encoder/wandb/run-20220530_120145-2k01y1ky +wandb: Run data is saved locally in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-feature-encoder/wandb/run-20220530_175928-2nzi0chc wandb: Run `wandb offline` to turn off syncing. wandb: Syncing run flax-wav2vec2-2-bart-large-ls-960h-feature-encoder wandb: ⭐️ View project at https://wandb.ai/sanchit-gandhi/librispeech_960h -wandb: 🚀 View run at https://wandb.ai/sanchit-gandhi/librispeech_960h/runs/2k01y1ky -05/30/2022 12:01:47 - INFO - __main__ - Training/evaluation parameters FlaxSeq2SeqTrainingArguments( +wandb: 🚀 View run at https://wandb.ai/sanchit-gandhi/librispeech_960h/runs/2nzi0chc +05/30/2022 17:59:29 - INFO - __main__ - Training/evaluation parameters FlaxSeq2SeqTrainingArguments( _n_gpu=-1, adafactor=False, adam_beta1=0.9, @@ -40,7 +40,7 @@ fp16_opt_level=O1, generation_length_penalty=1.2, generation_max_length=40, generation_num_beams=1, -gradient_accumulation_steps=1, +gradient_accumulation_steps=2, gradient_checkpointing=True, greater_is_better=None, group_by_length=False, @@ -76,7 +76,7 @@ output_dir=./, overwrite_output_dir=True, past_index=-1, per_device_eval_batch_size=4, -per_device_train_batch_size=8, +per_device_train_batch_size=4, precision=full, predict_with_generate=True, prediction_loss_only=False, @@ -105,16 +105,12 @@ warmup_steps=500, weight_decay=0.0, xpu_backend=None, ) -05/30/2022 12:01:47 - INFO - __main__ - JAX devices: 8, matmul precision: default -05/30/2022 12:01:47 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/sanchitgandhi/cache/huggingface/datasets/librispeech_asr/all/2.1.0/14c8bffddb861b4b3a4fcdff648a56980dbb808f3fc56f5a3d56b18ee88458eb) -05/30/2022 12:01:49 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/sanchitgandhi/cache/huggingface/datasets/librispeech_asr/all/2.1.0/14c8bffddb861b4b3a4fcdff648a56980dbb808f3fc56f5a3d56b18ee88458eb) -05/30/2022 12:01:50 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/sanchitgandhi/cache/huggingface/datasets/librispeech_asr/all/2.1.0/14c8bffddb861b4b3a4fcdff648a56980dbb808f3fc56f5a3d56b18ee88458eb) -05/30/2022 12:01:50 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/sanchitgandhi/cache/huggingface/datasets/librispeech_asr/all/2.1.0/14c8bffddb861b4b3a4fcdff648a56980dbb808f3fc56f5a3d56b18ee88458eb) -05/30/2022 12:01:51 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/sanchitgandhi/cache/huggingface/datasets/librispeech_asr/all/2.1.0/14c8bffddb861b4b3a4fcdff648a56980dbb808f3fc56f5a3d56b18ee88458eb) -https://huggingface.co/sanchit-gandhi/flax-wav2vec2-2-bart-large-scan/resolve/main/config.json not found in cache or force_download set to True, downloading to /home/sanchitgandhi/.cache/huggingface/transformers/tmp2_f644vh - Downloading: 0% 0.00/6.84k [00:00 + main() + File "run_flax_speech_recognition_seq2seq.py", line 1400, in main + state, train_metric = p_train_step(state, batch) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/_src/api.py", line 2013, in cache_miss + out_tree, out_flat = f_pmapped_(*args, **kwargs) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/_src/api.py", line 1889, in pmap_f + out = pxla.xla_pmap( + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/core.py", line 1863, in bind + return map_bind(self, fun, *args, **params) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/core.py", line 1895, in map_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/core.py", line 1866, in process + return trace.process_map(self, fun, tracers, params) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/core.py", line 660, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 801, in xla_pmap_impl + compiled_fun, fingerprint = parallel_callable( + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/linear_util.py", line 285, in memoized_fun + ans = call(fun, *args) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 829, in parallel_callable + pmap_computation = lower_parallel_callable( + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/_src/profiler.py", line 206, in wrapper + return func(*args, **kwargs) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1002, in lower_parallel_callable + jaxpr, consts, replicas, parts, shards = stage_parallel_callable( + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 909, in stage_parallel_callable + jaxpr, out_sharded_avals, consts = pe.trace_to_jaxpr_final( + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/_src/profiler.py", line 206, in wrapper + return func(*args, **kwargs) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/interpreters/partial_eval.py", line 1844, in trace_to_jaxpr_final + jaxpr, out_avals, consts = trace_to_subjaxpr_dynamic( + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/interpreters/partial_eval.py", line 1816, in trace_to_subjaxpr_dynamic + ans = fun.call_wrapped(*in_tracers_) + File "/home/sanchitgandhi/hf/lib/python3.8/site-packages/jax/linear_util.py", line 168, in call_wrapped + ans = self.f(*args, **dict(self.params, **kwargs)) + File "run_flax_speech_recognition_seq2seq.py", line 1174, in train_step + if gradient_accumulation_steps == 1 or training_args.multisteps: +jax._src.traceback_util.UnfilteredStackTrace: AttributeError: 'FlaxSeq2SeqTrainingArguments' object has no attribute 'multisteps' + +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. + +-------------------- -During handling of the above exception, another exception occurred: +The above exception was the direct cause of the following exception: Traceback (most recent call last): File "run_flax_speech_recognition_seq2seq.py", line 1507, in main() - File "run_flax_speech_recognition_seq2seq.py", line 949, in main - vectorized_datasets = raw_datasets.map( - File "/home/sanchitgandhi/datasets/src/datasets/dataset_dict.py", line 443, in map - { - File "/home/sanchitgandhi/datasets/src/datasets/dataset_dict.py", line 444, in - k: dataset.map( - File "/home/sanchitgandhi/datasets/src/datasets/arrow_dataset.py", line 1980, in map - return self._map_single( - File "/home/sanchitgandhi/datasets/src/datasets/arrow_dataset.py", line 521, in wrapper - out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs) - File "/home/sanchitgandhi/datasets/src/datasets/arrow_dataset.py", line 488, in wrapper - out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs) - File "/home/sanchitgandhi/datasets/src/datasets/fingerprint.py", line 458, in wrapper - out = func(self, *args, **kwargs) - File "/home/sanchitgandhi/datasets/src/datasets/arrow_dataset.py", line 2392, in _map_single - writer.finalize() - File "/home/sanchitgandhi/datasets/src/datasets/arrow_writer.py", line 537, in finalize - self.write_examples_on_file() - File "/home/sanchitgandhi/datasets/src/datasets/arrow_writer.py", line 414, in write_examples_on_file - self.write_batch(batch_examples=batch_examples) - File "/home/sanchitgandhi/datasets/src/datasets/arrow_writer.py", line 511, in write_batch - self.write_table(pa_table, writer_batch_size) - File "/home/sanchitgandhi/datasets/src/datasets/arrow_writer.py", line 528, in write_table - self.pa_writer.write_batch(batch) - File "pyarrow/ipc.pxi", line 429, in pyarrow.lib._CRecordBatchWriter.write_batch - File "pyarrow/error.pxi", line 112, in pyarrow.lib.check_status -OSError: [Errno 28] Error writing bytes to file. Detail: [errno 28] No space left on device + File "run_flax_speech_recognition_seq2seq.py", line 1400, in main + state, train_metric = p_train_step(state, batch) + File "run_flax_speech_recognition_seq2seq.py", line 1174, in train_step + if gradient_accumulation_steps == 1 or training_args.multisteps: +AttributeError: 'FlaxSeq2SeqTrainingArguments' object has no attribute 'multisteps' wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. -wandb: - 0.002 MB of 0.002 MB uploaded (0.000 MB deduped) wandb: \ 0.002 MB of 0.002 MB uploaded (0.000 MB deduped) wandb: | 0.002 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: / 0.002 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: - 0.018 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: \ 0.018 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: | 0.021 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: / 0.021 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: - 0.021 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: \ 0.021 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: | 0.021 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: / 0.021 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: - 0.021 MB of 0.021 MB uploaded (0.000 MB deduped) wandb: -wandb: Synced flax-wav2vec2-2-bart-large-ls-960h-feature-encoder: https://wandb.ai/sanchit-gandhi/librispeech_960h/runs/2k01y1ky +wandb: - 0.002 MB of 0.002 MB uploaded (0.000 MB deduped) wandb: \ 0.002 MB of 0.002 MB uploaded (0.000 MB deduped) wandb: | 0.002 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: / 0.002 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: - 0.034 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: \ 0.034 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: | 0.034 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: / 0.034 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: - 0.034 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: \ 0.034 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: | 0.034 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: / 0.034 MB of 0.034 MB uploaded (0.000 MB deduped) wandb: +wandb: Synced flax-wav2vec2-2-bart-large-ls-960h-feature-encoder: https://wandb.ai/sanchit-gandhi/librispeech_960h/runs/2nzi0chc wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) -wandb: Find logs at: ./wandb/run-20220530_120145-2k01y1ky/logs +wandb: Find logs at: ./wandb/run-20220530_175928-2nzi0chc/logs