AttributeError: 'HfTrainerDeepSpeedConfig' object has no attribute 'is_zero3'
Author: JaydenChao101
Pull request: no
Labels: name: bug

### System Info

- `transformers` version: 4.53.0
- Platform: Linux-6.1.123+-x86_64-with-glibc2.35
- Python version: 3.11.13
- Huggingface_hub version: 0.33.1
- Safetensors version: 0.5.3
- Accelerate version: 1.8.1
- Accelerate config: 	not found
- DeepSpeed version: not installed
- PyTorch version (accelerator?): 2.7.0+cu126 (CUDA)
- Tensorflow version (GPU?): 2.18.0 (True)
- Flax version (CPU?/GPU?/TPU?): 0.10.6 (gpu)
- Jax version: 0.5.2
- JaxLib version: 0.5.1
- Using distributed or parallel set-up in script?: <fill in>
- Using GPU in script?: Yes
- GPU type: NVIDIA A100-SXM4-40GB

### Who can help?

@SunMarc @zach-huggingface

### Information

- [ ] The official example scripts
- [x] My own modified scripts

### Tasks

- [x] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction
```python
# 1. 掛載 Google Drive (如果您在 Colab 環境)
from google.colab import drive
import os
import shutil
import time
import threading
import gc
import torch
import wandb

# 確保乾淨的環境
try:
    drive.flush_and_unmount()
except Exception as e:
    print(f"Drive unmount failed (this is expected if not mounted): {e}")
drive.mount('/content/drive', force_remount=True)

# 清理 CUDA 快取
gc.collect()
torch.cuda.empty_cache()

# 2. 設定路徑
# 訓練期間，Checkpoint 會先存在本機以提高速度
local_ckpt_dir = "/content/qwen3-8b-finetuned-local"
# 雲端硬碟的備份路徑
drive_ckpt_dir = "/content/drive/MyDrive/TinyQwen3-4B-GC-Math-full"
os.makedirs(local_ckpt_dir, exist_ok=True)
os.makedirs(drive_ckpt_dir, exist_ok=True)

# ==============================================================================
# 2. 同步功能：非同步將 Checkpoint 備份到 Google Drive
# ==============================================================================
def sync_checkpoints_to_drive():
    """
    將本地端最新的 checkpoint 完整複製到 Google Drive。
    這是一個阻塞操作，會先刪除舊的備份再複製新的。
    """
    print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting sync to Drive...")
    try:
        # 使用 shutil.rmtree 刪除舊的備份
        if os.path.exists(drive_ckpt_dir):
            shutil.rmtree(drive_ckpt_dir)
            print(f"Removed old Drive backup at: {drive_ckpt_dir}")
        # 使用 shutil.copytree 複製整個目錄樹
        shutil.copytree(local_ckpt_dir, drive_ckpt_dir)
        print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Sync to Drive successful!")
    except Exception as e:
        print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR during sync: {e}")

def async_sync_checkpoints():
    """
    使用單獨的執行緒在背景執行同步任務，避免中斷訓練。
    """
    sync_thread = threading.Thread(target=sync_checkpoints_to_drive)
    sync_thread.daemon = True # 確保主程式結束時執行緒也會結束
    sync_thread.start()
    print("Background sync process initiated.")

# ==============================================================================
# 3. 載入模型與 Tokenizer (A100 優化)
# ==============================================================================
from unsloth import FastLanguageModel
# *** A100 優化 ***
# 使用 Unsloth 優化的 bf16 模型，不進行 4-bit 量化
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name = "TinyQwen/TinyQwen3-4B-GC", # 使用 Unsloth 優化的 bf16 版本
    max_seq_length = 2048,
    dtype = torch.bfloat16, # 為 A100 指定 bfloat16
    load_in_4bit = False,   # 關閉 4-bit 量化
    attn_implementation = "flash_attention_2", # 使用 Flash Attention 2 加速
)

# ==============================================================================
# 4. 設定 LoRA (PEFT) (A100 優化)
# ==============================================================================
# 添加結構化推理用的特殊 token
tokenizer.add_tokens(["<reasoning>", "</reasoning>", "<answer>", "</answer>"], special_tokens=True)
# 調整模型的 token embedding 層以適應新的 token
model.resize_token_embeddings(len(tokenizer))

model = FastLanguageModel.get_peft_model(
    model,
    r = 32, # *** A100 優化：提升 LoRA rank，增加模型容量 ***
    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
    lora_alpha = 32,
    lora_dropout = 0.05,
    bias = "none",
    use_gradient_checkpointing = False,
)

# ==============================================================================
# 5. 讀取並處理資料集
# ==============================================================================
from datasets import load_dataset
import pandas as pd
from datasets import Dataset

# 從 Hugging Face Hub 下載資料集
dataset = load_dataset("open-r1/Mixture-of-Thoughts", "code", split="train[:50000]")

# 定義一個函數，將所有需要的資訊格式化成單一的文字欄位 'text'
def format_prompt(example):
  msgs = example.get("messages") or example.get("conversations") or []
  text = ""
  for m in msgs:
    role = m.get("role")
    content = m.get("content", "").strip()
    if role in ["user", "human"]:
      text += f"Human: {content}\n"
    elif role == "assistant":
      text += f"Assistant: {content}\n"
  return {"text": text.strip()}

# 使用 .map() 應用格式化函數，創建 'text' 欄位並移除其他欄位
dataset = dataset.map(format_prompt, remove_columns=dataset.column_names)

# 切分訓練集和驗證集
dataset = dataset.train_test_split(test_size=0.01, seed=42)
train_ds = dataset["train"]
eval_ds = dataset["test"]
print(f"資料處理完成。訓練集大小: {len(train_ds)}, 驗證集大小: {len(eval_ds)}")
print("\n處理後的一筆資料範例:")
print(train_ds[0]['text'])

# ==============================================================================
# 6. 設定訓練參數與 Callback (A100 優化)
# ==============================================================================
from transformers import TrainingArguments, TrainerCallback, TrainerState, TrainerControl
from trl import SFTTrainer

class DriveSyncCallback(TrainerCallback):
    def on_save(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):
        print(f"\nCheckpoint saved at step {state.global_step}. Triggering background sync to Drive.")
        async_sync_checkpoints()
        return control

training_args = TrainingArguments(
    output_dir=local_ckpt_dir,
    overwrite_output_dir=True,
    num_train_epochs=3,
    per_device_train_batch_size=16, # *** A100 優化：大幅提升批次大小 ***
    gradient_accumulation_steps=2,  # 有效 batch size = 16 * 2 = 32
    learning_rate=2e-4,
    save_strategy="steps",
    save_steps=200,
    eval_steps=100,
    save_total_limit=2,
    logging_steps=25,
    bf16=True,                      # 啟用 bfloat16
    report_to="wandb",
    #deepspeed="./ds_config.json",   # ← 啟用 DeepSpeed ZeRO
    remove_unused_columns=False,
)

# ==============================================================================
# 7. 初始化並開始訓練
# ==============================================================================
trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    args=training_args,
    train_dataset=train_ds,
    eval_dataset=eval_ds,
    dataset_text_field="text",
    max_seq_length=2048,
    packing=False,
    callbacks=[DriveSyncCallback()],
)

# 使用 try...finally 確保訓練中斷時也能保存最終模型
try:
    print("="*20 + " 開始訓練 (A100 優化模式) " + "="*20)
    latest_checkpoint = None
    if os.path.exists(local_ckpt_dir):
        checkpoints = [d for d in os.listdir(local_ckpt_dir) if d.startswith("checkpoint-")]
        if checkpoints:
            latest_checkpoint = os.path.join(local_ckpt_dir, max(checkpoints, key=lambda x: int(x.split('-')[-1])))
            print(f"發現可用的 Checkpoint，將從 {latest_checkpoint} 恢復訓練。")

    trainer.train(resume_from_checkpoint=latest_checkpoint)

except Exception as e:
    print(f"\n訓練過程中發生錯誤: {e}")

finally:
    print("\n訓練結束或中斷。正在保存最終的 LoRA adapter 並執行最後一次同步...")
    final_adapter_dir = os.path.join(local_ckpt_dir, "final_adapter")
    model.save_pretrained(final_adapter_dir)
    tokenizer.save_pretrained(final_adapter_dir)
    print(f"最終 adapter 已保存至: {final_adapter_dir}")
    sync_checkpoints_to_drive()
    print("所有任務完成。")


# 10. 清理
gc.collect()
torch.cuda.empty_cache()

from google.colab import runtime
runtime.unassign()
```

### Expected behavior

I expected the TrainingArguments(deepspeed=...) integration with the Hugging Face Trainer to work without raising an AttributeError related to HfTrainerDeepSpeedConfig.

Specifically, the method is_zero3() used inside the integration code should either:

- Still exist and be accessible, or
- Be safely replaced/removed in all internal references when removed in recent versions.

Comments:
This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) are likely to be ignored.