Repository: huggingface/trl

Issue:
Error when run `trl/examples/research_projects/stack_llama_2/scripts /sft_llama2.py`
I tried to run the stack_llama_2 project and encountered the following error.

The command I used:

```
accelerate launch sft_llama2.py --training_args.output_dir="sft"
```
The error is as folloing:
```
[2023-11-24 03:28:10,125] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect)
/usr/local/lib/python3.10/dist-packages/trl/trainer/ppo_config.py:141: UserWarning: The `optimize_cuda_cache` arguement will be deprecated soon, please use `optimize_device_cache` instead.
  warnings.warn(
/usr/local/lib/python3.10/dist-packages/tyro/_resolver.py:311: UserWarning: <class 'int'> does not match any type in Union: [<class 'float'>, <class 'NoneType'>]
  warnings.warn(
/usr/local/lib/python3.10/dist-packages/tyro/_resolver.py:311: UserWarning: <class 'dict'> does not match any type in Union: [<class 'str'>, <class 'NoneType'>]
  warnings.warn(
Traceback (most recent call last):
  File "/home/trl/examples/research_projects/stack_llama_2/scripts/sft_llama2.py", line 69, in <module>
    script_args = tyro.cli(ScriptArguments)
  File "/usr/local/lib/python3.10/dist-packages/tyro/_cli.py", line 187, in cli
    output = _cli_impl(
  File "/usr/local/lib/python3.10/dist-packages/tyro/_cli.py", line 374, in _cli_impl
    parser_spec = _parsers.ParserSpecification.from_callable_or_type(
  File "/usr/local/lib/python3.10/dist-packages/tyro/_parsers.py", line 106, in from_callable_or_type
    field_out = handle_field(
  File "/usr/local/lib/python3.10/dist-packages/tyro/_parsers.py", line 320, in handle_field
    return ParserSpecification.from_callable_or_type(
  File "/usr/local/lib/python3.10/dist-packages/tyro/_parsers.py", line 106, in from_callable_or_type
    field_out = handle_field(
  File "/usr/local/lib/python3.10/dist-packages/tyro/_parsers.py", line 312, in handle_field
    if _fields.is_nested_type(field.type_or_callable, field.default):
  File "/usr/local/lib/python3.10/dist-packages/tyro/_unsafe_cache.py", line 33, in wrapped_f
    out = f(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/tyro/_fields.py", line 224, in is_nested_type
    _try_field_list_from_callable(typ, default_instance),
  File "/usr/local/lib/python3.10/dist-packages/tyro/_fields.py", line 348, in _try_field_list_from_callable
    return _field_list_from_dict(f, default_instance)
  File "/usr/local/lib/python3.10/dist-packages/tyro/_fields.py", line 734, in _field_list_from_dict
    if default_instance in MISSING_SINGLETONS or len(cast(dict, default_instance)) == 0:
TypeError: object of type 'NoneType' has no len()
Traceback (most recent call last):
  File "/usr/local/bin/accelerate", line 8, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.10/dist-packages/accelerate/commands/accelerate_cli.py", line 47, in main
    args.func(args)
  File "/usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py", line 994, in launch_command
    simple_launcher(args)
  File "/usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py", line 636, in simple_launcher
    raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
subprocess.CalledProcessError: Command '['/usr/bin/python', 'sft_llama2.py', '--training_args.output_dir=sft']' returned non-zero exit status 1.
```
It seems that line 69 of sft_llama2.py `script_args = tyro.cli(ScriptArguments)` leads to the error `TypeError: object of type 'NoneType' has no len()`. 
I am not familiar with tyro, could anyone help me with this problem? Any suggestions would be appreciated!


Discussion:
cc @vwxyzjn 
Please update the `transformers` (`pip install --upgrade transformers`). This is a duplicate issue of https://github.com/huggingface/trl/pull/906#issuecomment-1815312980. 
I still have no idea why to import 'tyro'...

It throws another error:

  File "/home/nus_ids_user3/Projects/ultramarine/src/sft_train.py", line 71, in <module>
    script_args = tyro.cli(ScriptArguments)
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_cli.py", line 187, in cli
    output = _cli_impl(
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_cli.py", line 374, in _cli_impl
    parser_spec = _parsers.ParserSpecification.from_callable_or_type(
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_parsers.py", line 110, in from_callable_or_type
    field_out = handle_field(
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_parsers.py", line 322, in handle_field
    if _fields.is_nested_type(field.type_or_callable, field.default):
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_unsafe_cache.py", line 33, in wrapped_f
    out = f(*args, **kwargs)
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_fields.py", line 249, in is_nested_type
    _try_field_list_from_callable(typ, default_instance),
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_fields.py", line 412, in _try_field_list_from_callable
    return field_list_from_class(cls, default_instance)
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_fields.py", line 559, in _field_list_from_dataclass
    for dc_field in filter(lambda field: field.init, _resolver.resolved_fields(cls)):
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_unsafe_cache.py", line 33, in wrapped_f
    out = f(*args, **kwargs)
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/tyro/_resolver.py", line 97, in resolved_fields
    annotations = get_type_hints(cls, include_extras=True)
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/site-packages/typing_extensions.py", line 1266, in get_type_hints
    hint = typing.get_type_hints(
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/typing.py", line 1459, in get_type_hints
    value = _eval_type(value, base_globals, localns)
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/typing.py", line 292, in _eval_type
    return t._evaluate(globalns, localns, recursive_guard)
  File "/mnt/dataDisk1/miniconda3/envs/mercury/lib/python3.9/typing.py", line 554, in _evaluate
    eval(self.__forward_code__, globalns, localns),
  File "<string>", line 1, in <module>
TypeError: unsupported operand type(s) for |: 'type' and '_LiteralGenericAlias'

Resolving patch:
diff --git a/examples/research_projects/stack_llama_2/scripts/sft_llama2.py b/examples/research_projects/stack_llama_2/scripts/sft_llama2.py
--- a/examples/research_projects/stack_llama_2/scripts/sft_llama2.py
+++ b/examples/research_projects/stack_llama_2/scripts/sft_llama2.py
@@ -4,12 +4,11 @@
 from typing import Optional
 
 import torch
-import tyro
 from accelerate import Accelerator
 from datasets import load_dataset
 from peft import AutoPeftModelForCausalLM, LoraConfig
 from tqdm import tqdm
-from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TrainingArguments
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, HfArgumentParser, TrainingArguments
 
 from trl import SFTTrainer
 from trl.import_utils import is_xpu_available
@@ -19,7 +18,6 @@
 @dataclass
 class ScriptArguments:
     model_name: Optional[str] = field(default="meta-llama/Llama-2-7b-hf", metadata={"help": "the model name"})
-
     dataset_name: Optional[str] = field(default="lvwerra/stack-exchange-paired", metadata={"help": "the dataset name"})
     subset: Optional[str] = field(default="data/finetune", metadata={"help": "the subset to use"})
     split: Optional[str] = field(default="train", metadata={"help": "the split to use"})
@@ -28,52 +26,31 @@ class ScriptArguments:
     shuffle_buffer: Optional[int] = field(default=5000, metadata={"help": "the shuffle buffer size"})
     seq_length: Optional[int] = field(default=1024, metadata={"help": "the sequence length"})
     num_workers: Optional[int] = field(default=4, metadata={"help": "the number of workers"})
-
-    training_args: TrainingArguments = field(
-        default_factory=lambda: TrainingArguments(
-            output_dir="./results",
-            max_steps=500,
-            logging_steps=10,
-            save_steps=10,
-            per_device_train_batch_size=4,
-            per_device_eval_batch_size=1,
-            gradient_accumulation_steps=2,
-            gradient_checkpointing=False,
-            group_by_length=False,
-            learning_rate=1e-4,
-            lr_scheduler_type="cosine",
-            warmup_steps=100,
-            weight_decay=0.05,
-            optim="paged_adamw_32bit",
-            bf16=True,
-            remove_unused_columns=False,
-            run_name="sft_llama2",
-            report_to="wandb",
-        )
-    )
-
     packing: Optional[bool] = field(default=True, metadata={"help": "whether to use packing for SFTTrainer"})
 
-    peft_config: LoraConfig = field(
-        default_factory=lambda: LoraConfig(
-            r=8,
-            lora_alpha=16,
-            lora_dropout=0.05,
-            target_modules=["q_proj", "v_proj"],
-            bias="none",
-            task_type="CAUSAL_LM",
-        )
-    )
-
-
-script_args = tyro.cli(ScriptArguments)
+    # LoraConfig
+    lora_alpha: Optional[float] = field(default=16, metadata={"help": "the lora alpha parameter"})
+    lora_dropout: Optional[float] = field(default=0.05, metadata={"help": "the lora dropout parameter"})
+    lora_r: Optional[int] = field(default=8, metadata={"help": "the lora r parameter"})
+
+
+parser = HfArgumentParser((ScriptArguments, TrainingArguments))
+script_args, training_args = parser.parse_args_into_dataclasses()
+peft_config = LoraConfig(
+    r=script_args.lora_r,
+    lora_alpha=script_args.lora_alpha,
+    lora_dropout=script_args.lora_dropout,
+    target_modules=["q_proj", "v_proj"],
+    bias="none",
+    task_type="CAUSAL_LM",
+)
 
-if script_args.training_args.group_by_length and script_args.packing:
+if training_args.group_by_length and script_args.packing:
     raise ValueError("Cannot use both packing and group by length")
 
 # `gradient_checkpointing` was True by default until `1f3314`, but it's actually not used.
 # `gradient_checkpointing=True` will cause `Variable._execution_engine.run_backward`.
-if script_args.training_args.gradient_checkpointing:
+if training_args.gradient_checkpointing:
     raise ValueError("gradient_checkpointing not supported")
 
 
@@ -171,14 +148,11 @@ def create_datasets(tokenizer, args):
 )
 base_model.config.use_cache = False
 
-peft_config = script_args.peft_config
 
 tokenizer = AutoTokenizer.from_pretrained(script_args.model_name, trust_remote_code=True)
 tokenizer.pad_token = tokenizer.eos_token
 tokenizer.padding_side = "right"  # Fix weird overflow issue with fp16 training
 
-training_args = script_args.training_args
-
 train_dataset, eval_dataset = create_datasets(tokenizer, script_args)
 
 trainer = SFTTrainer(
@@ -192,9 +166,9 @@ def create_datasets(tokenizer, args):
     args=training_args,
 )
 trainer.train()
-trainer.save_model(script_args.training_args.output_dir)
+trainer.save_model(training_args.output_dir)
 
-output_dir = os.path.join(script_args.training_args.output_dir, "final_checkpoint")
+output_dir = os.path.join(training_args.output_dir, "final_checkpoint")
 trainer.model.save_pretrained(output_dir)
 
 # Free memory for merging weights
@@ -207,5 +181,5 @@ def create_datasets(tokenizer, args):
 model = AutoPeftModelForCausalLM.from_pretrained(output_dir, device_map="auto", torch_dtype=torch.bfloat16)
 model = model.merge_and_unload()
 
-output_merged_dir = os.path.join(script_args.training_args.output_dir, "final_merged_checkpoint")
+output_merged_dir = os.path.join(training_args.output_dir, "final_merged_checkpoint")
 model.save_pretrained(output_merged_dir, safe_serialization=True)


Test patch: