versae commited on
Commit
a95f7b8
1 Parent(s): 40f69ff

Changed batch size

Browse files
Files changed (2) hide show
  1. run.sh +9 -3
  2. run_mlm_flax.py +4 -4
run.sh CHANGED
@@ -6,10 +6,16 @@
6
  --dataset_name="large_spanish_corpus" \
7
  --dataset_config_name="combined" \
8
  --max_seq_length="128" \
9
- --per_device_train_batch_size="4" \
10
- --per_device_eval_batch_size="4" \
 
11
  --learning_rate="3e-4" \
 
 
 
12
  --warmup_steps="1000" \
13
  --overwrite_output_dir \
14
  --num_train_epochs="8" \
15
- --push_to_hub
 
 
 
6
  --dataset_name="large_spanish_corpus" \
7
  --dataset_config_name="combined" \
8
  --max_seq_length="128" \
9
+ --pad_to_max_length \
10
+ --per_device_train_batch_size="128" \
11
+ --per_device_eval_batch_size="128" \
12
  --learning_rate="3e-4" \
13
+ --save_strategy="steps" \
14
+ --save_steps="10000" \
15
+ --save_total_limit="5" \
16
  --warmup_steps="1000" \
17
  --overwrite_output_dir \
18
  --num_train_epochs="8" \
19
+ --dtype="bfloat16" \
20
+ --push_to_hub_model_id="flax-community/bertin-roberta-large-spanish" \
21
+ --push_to_hub 2>&1 | tee run.log
run_mlm_flax.py CHANGED
@@ -315,10 +315,10 @@ if __name__ == "__main__":
315
 
316
  # Log on each process the small summary:
317
  logger = logging.getLogger(__name__)
318
- logger.warning(
319
- f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
320
- + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
321
- )
322
 
323
  # Set the verbosity to info of the Transformers logger (on main process only):
324
  logger.info(f"Training/evaluation parameters {training_args}")
 
315
 
316
  # Log on each process the small summary:
317
  logger = logging.getLogger(__name__)
318
+ #logger.warning(
319
+ # f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
320
+ # + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
321
+ #)
322
 
323
  # Set the verbosity to info of the Transformers logger (on main process only):
324
  logger.info(f"Training/evaluation parameters {training_args}")