Training in progress, epoch 1

eval_job_output.txt

@@ -1,4 +1,4 @@
-slurm submission log: 2024-05-22 17:07:25.162212
+slurm submission log: 2024-05-23 14:58:53.803713
 created following sbatch script: 
 
 ###############################
@@ -7,13 +7,13 @@ created following sbatch script:
 
 #SBATCH --account=nlp
 #SBATCH --cpus-per-task=16
-#SBATCH --dependency=afterok:7642740
+#SBATCH --dependency=afterok:7645740
 #SBATCH --gres=gpu:1
-#SBATCH --job-name=tthrush-job-2791360
+#SBATCH --job-name=tthrush-job-1104501
 #SBATCH --mem=60G
 #SBATCH --nodelist=sphinx1
 #SBATCH --open-mode=append
-#SBATCH --output=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/eval_job_output.txt
+#SBATCH --output=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained_initial_init/llms/pythia-70m_sciq_1/eval_job_output.txt
 #SBATCH --partition=sphinx
 #SBATCH --time=14-0
 
@@ -24,7 +24,7 @@ created following sbatch script:
 cd .
 
 # launch commands
-srun --unbuffered run_as_child_processes 'lm_eval --model hf --model_args pretrained=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1,revision=main,dtype=float16,trust_remote_code=True --tasks xnli_en,xnli_fr,sciq,piqa,lambada,arc_easy --device cuda --output_path /juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/perf'
+srun --unbuffered run_as_child_processes 'lm_eval --model hf --model_args pretrained=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained_initial_init/llms/pythia-70m_sciq_1,revision=main,dtype=float16,trust_remote_code=True --tasks xnli_en,xnli_fr,sciq,piqa,lambada,arc_easy --device cuda --output_path /juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained_initial_init/llms/pythia-70m_sciq_1/perf'
 
 ###############################
 
@@ -34,249 +34,7 @@ submission to slurm complete!
 ###############################
 slurm submission output
 
-Submitted batch job 7642741
-
-
-
-###############################
-
-slurm submission log: 2024-05-22 17:23:51.579657
-created following sbatch script: 
-
-###############################
-
-#!/bin/bash
-
-#SBATCH --account=nlp
-#SBATCH --cpus-per-task=16
-#SBATCH --dependency=afterok:7642780
-#SBATCH --gres=gpu:1
-#SBATCH --job-name=tthrush-job-1978619
-#SBATCH --mem=60G
-#SBATCH --nodelist=sphinx1
-#SBATCH --open-mode=append
-#SBATCH --output=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/eval_job_output.txt
-#SBATCH --partition=sphinx
-#SBATCH --time=14-0
-
-# activate your desired anaconda environment
-. /nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/etc/profile.d/conda.sh ; conda activate pretraining-coreset-selection
-
-# cd to working directory
-cd .
-
-# launch commands
-srun --unbuffered run_as_child_processes 'lm_eval --model hf --model_args pretrained=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1,revision=main,dtype=float16,trust_remote_code=True --tasks xnli_en,xnli_fr,sciq,piqa,lambada,arc_easy --device cuda --output_path /juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/perf'
-
-###############################
-
-submission to slurm complete!
-
-
-###############################
-slurm submission output
-
-Submitted batch job 7642781
-
-
-
-###############################
-
-slurm submission log: 2024-05-22 17:29:15.965569
-created following sbatch script: 
-
-###############################
-
-#!/bin/bash
-
-#SBATCH --account=nlp
-#SBATCH --cpus-per-task=16
-#SBATCH --dependency=afterok:7642805
-#SBATCH --gres=gpu:1
-#SBATCH --job-name=tthrush-job-2597090
-#SBATCH --mem=60G
-#SBATCH --nodelist=sphinx1
-#SBATCH --open-mode=append
-#SBATCH --output=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/eval_job_output.txt
-#SBATCH --partition=sphinx
-#SBATCH --time=14-0
-
-# activate your desired anaconda environment
-. /nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/etc/profile.d/conda.sh ; conda activate pretraining-coreset-selection
-
-# cd to working directory
-cd .
-
-# launch commands
-srun --unbuffered run_as_child_processes 'lm_eval --model hf --model_args pretrained=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1,revision=main,dtype=float16,trust_remote_code=True --tasks xnli_en,xnli_fr,sciq,piqa,lambada,arc_easy --device cuda --output_path /juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/perf'
-
-###############################
-
-submission to slurm complete!
-
-
-###############################
-slurm submission output
-
-Submitted batch job 7642806
-
-
-
-###############################
-
-/var/lib/slurm/slurmd/job7642806/slurm_script: line 16: /nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/etc/profile.d/conda.sh: No such file or directory
-
-CommandNotFoundError: Your shell has not been properly configured to use 'conda activate'.
-To initialize your shell, run
-
-    $ conda init <SHELL_NAME>
-
-Currently supported shells are:
-  - bash
-  - fish
-  - tcsh
-  - xonsh
-  - zsh
-  - powershell
-
-See 'conda init --help' for more information and options.
-
-IMPORTANT: You may need to close and restart your shell after running 'conda init'.
-
-
-###############################
-start time: 2024-05-22 17:31:53.790973
-machine: sphinx1
-conda env: pretraining-coreset-selection
-###############################
-running following processes
-
-	lm_eval --model hf --model_args pretrained=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1,revision=main,dtype=float16,trust_remote_code=True --tasks xnli_en,xnli_fr,sciq,piqa,lambada,arc_easy --device cuda --output_path /juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/perf
-
-
-###############################
-command outputs: 
-
-
-2024-05-22:17:31:59,220 INFO     [utils.py:145] Note: detected 255 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
-2024-05-22:17:31:59,220 INFO     [utils.py:148] Note: NumExpr detected 255 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 8.
-2024-05-22:17:31:59,220 INFO     [utils.py:160] NumExpr defaulting to 8 threads.
-2024-05-22:17:31:59,893 INFO     [config.py:58] PyTorch version 2.2.2 available.
-2024-05-22:17:32:04,203 INFO     [__main__.py:156] Verbosity set to INFO
-2024-05-22:17:32:14,116 WARNING  [__init__.py:194] Some tasks could not be loaded due to missing dependencies. Run with `--verbosity DEBUG` for full details.
-/nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/lib/python3.10/site-packages/datasets/load.py:1429: FutureWarning: The repository for hails/mmlu_no_train contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/hails/mmlu_no_train
-You can avoid this message in future by passing the argument `trust_remote_code=True`.
-Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.
-  warnings.warn(
-2024-05-22:17:33:33,299 WARNING  [__init__.py:194] Some tasks could not be loaded due to missing dependencies. Run with `--verbosity DEBUG` for full details.
-2024-05-22:17:33:33,305 INFO     [__main__.py:229] Selected Tasks: ['arc_easy', 'lambada', 'piqa', 'sciq', 'xnli_en', 'xnli_fr']
-2024-05-22:17:33:33,714 INFO     [huggingface.py:148] Using device 'cuda'
-Traceback (most recent call last):
-  File "/nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/bin/lm_eval", line 8, in <module>
-    sys.exit(cli_evaluate())
-  File "/sailhome/tthrush/lm-evaluation-harness/lm_eval/__main__.py", line 231, in cli_evaluate
-    results = evaluator.simple_evaluate(
-  File "/sailhome/tthrush/lm-evaluation-harness/lm_eval/utils.py", line 415, in _wrapper
-    return fn(*args, **kwargs)
-  File "/sailhome/tthrush/lm-evaluation-harness/lm_eval/evaluator.py", line 98, in simple_evaluate
-    lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
-  File "/sailhome/tthrush/lm-evaluation-harness/lm_eval/api/model.py", line 134, in create_from_arg_string
-    return cls(**args, **args2)
-  File "/sailhome/tthrush/lm-evaluation-harness/lm_eval/models/huggingface.py", line 174, in __init__
-    self._get_config(
-  File "/sailhome/tthrush/lm-evaluation-harness/lm_eval/models/huggingface.py", line 420, in _get_config
-    self._config = transformers.AutoConfig.from_pretrained(
-  File "/nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/lib/python3.10/site-packages/transformers/models/auto/configuration_auto.py", line 928, in from_pretrained
-    config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
-  File "/nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/lib/python3.10/site-packages/transformers/configuration_utils.py", line 631, in get_config_dict
-    config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
-  File "/nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/lib/python3.10/site-packages/transformers/configuration_utils.py", line 686, in _get_config_dict
-    resolved_config_file = cached_file(
-  File "/nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/lib/python3.10/site-packages/transformers/utils/hub.py", line 369, in cached_file
-    raise EnvironmentError(
-OSError: /juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1 does not appear to have a file named config.json. Checkout 'https://huggingface.co//juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/tree/main' for available files.
-###############################
-end time: 2024-05-22 17:33:43.911118
-elapsed time: 0:01:50.120145
-slurm submission log: 2024-05-22 17:41:39.457215
-created following sbatch script: 
-
-###############################
-
-#!/bin/bash
-
-#SBATCH --account=nlp
-#SBATCH --cpus-per-task=16
-#SBATCH --dependency=afterok:7642834
-#SBATCH --gres=gpu:1
-#SBATCH --job-name=tthrush-job-24240
-#SBATCH --mem=60G
-#SBATCH --nodelist=sphinx1
-#SBATCH --open-mode=append
-#SBATCH --output=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/eval_job_output.txt
-#SBATCH --partition=sphinx
-#SBATCH --time=14-0
-
-# activate your desired anaconda environment
-. /nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/etc/profile.d/conda.sh ; conda activate pretraining-coreset-selection
-
-# cd to working directory
-cd .
-
-# launch commands
-srun --unbuffered run_as_child_processes 'lm_eval --model hf --model_args pretrained=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1,revision=main,dtype=float16,trust_remote_code=True --tasks xnli_en,xnli_fr,sciq,piqa,lambada,arc_easy --device cuda --output_path /juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/perf'
-
-###############################
-
-submission to slurm complete!
-
-
-###############################
-slurm submission output
-
-Submitted batch job 7642835
-
-
-
-###############################
-
-slurm submission log: 2024-05-22 19:52:23.060911
-created following sbatch script: 
-
-###############################
-
-#!/bin/bash
-
-#SBATCH --account=nlp
-#SBATCH --cpus-per-task=16
-#SBATCH --dependency=afterok:7643057
-#SBATCH --gres=gpu:1
-#SBATCH --job-name=tthrush-job-4137796
-#SBATCH --mem=60G
-#SBATCH --nodelist=sphinx1
-#SBATCH --open-mode=append
-#SBATCH --output=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/eval_job_output.txt
-#SBATCH --partition=sphinx
-#SBATCH --time=14-0
-
-# activate your desired anaconda environment
-. /nlp/scr/tthrush/miniconda3/envs/pretraining-coreset-selection/etc/profile.d/conda.sh ; conda activate pretraining-coreset-selection
-
-# cd to working directory
-cd .
-
-# launch commands
-srun --unbuffered run_as_child_processes 'lm_eval --model hf --model_args pretrained=/juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1,revision=main,dtype=float16,trust_remote_code=True --tasks xnli_en,xnli_fr,sciq,piqa,lambada,arc_easy --device cuda --output_path /juice5/scr5/tthrush/pretraining-coreset-selection/llm_pretraining/test_ordinal_constrained/llms/pythia-70m_sciq_1/perf'
-
-###############################
-
-submission to slurm complete!
-
-
-###############################
-slurm submission output
-
-Submitted batch job 7643058
+Submitted batch job 7645741
 
 
 

logs/events.out.tfevents.1716527049.sphinx2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cd3605c757a4df33c8a571d76311882aafb9dd1234da5cf769c80b69c8c3b68
+size 95678

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d1d0b5f6fe2fd1b3e778eb06905f9ff0b46cd859dd0d2696a8fdd8dfb3af932
+oid sha256:72d975c1a09895fb677a80c6976a87b7c0c808aff25c8bd8eea46a1f10e6607c
 size 281715176

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42604c6e8628be2643fe1460d9bd416ac8e71af2f5f0e7182a340f5cb4e9907f
-size 5176
+oid sha256:6e5ec4dcc9a1a5c561e3555297a62670552352ebb9dca8bbc21575d63cf52a8c
+size 5240