How to fix this error:
```
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
import torch
import os
import json
import pandas as pd
from datasets import Dataset
from transformers import (
    AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer,
    DataCollatorForLanguageModeling, pipeline, BitsAndBytesConfig
)
import random
import numpy as np

# ------------- Reproducibility Setup -------------
SEED = 42
random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
if torch.cuda.is_available():
    torch.cuda.manual_seed_all(SEED)

# -----------------------------
# Step 1: Load and preprocess data
# -----------------------------
with open("balanced_labeled_conversations.json", "r") as f:
    data = json.load(f)

df = pd.DataFrame(data)
traits = ["witty", "sarcastic", "charming"]
df = df[df['trait'].isin(traits)].dropna(subset=["dialogue"])

def format_conversation_with_expert_token(row):
    expert_token = f"[EXPERT: {row['trait'].upper()}]"
    return f"{expert_token} 
 {row['dialogue']}\n
"

df["text"] = df.apply(format_conversation_with_expert_token, axis=1)
df = df.sample(frac=0.1, random_state=SEED).reset_index(drop=True)
print(f"Using {len(df)} examples ({len(df)/len(data):.0%} of original)")

# -----------------------------
# Step 2: Tokenization and special token addition
# -----------------------------
model_checkpoint = "deepseek-ai/deepseek-moe-16b-base"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, trust_remote_code=True)

# Add expert control tokens explicitly
special_tokens = [f"[EXPERT: {trait.upper()}]" for trait in traits]
num_added = tokenizer.add_tokens(special_tokens)
print(f"Added {num_added} special tokens: {special_tokens}")

# Ensure pad_token is defined
if tokenizer.pad_token is None:
    tokenizer.pad_token = tokenizer.eos_token

def tokenize_data(batch):
    return tokenizer(
        batch["text"],
        padding="max_length",
        truncation=True,
        max_length=512,
    )

# Build Dataset from just the sampled df
dataset = Dataset.from_pandas(df[["text"]])
tokenized_dataset = dataset.map(tokenize_data, batched=True, remove_columns=["text"])

# -----------------------------
# Step 3: Load model and resize embeddings
# -----------------------------

# Configure 8-bit quantization
bnb_config = BitsAndBytesConfig(
    load_in_8bit=True,
    bnb_8bit_compute_dtype=torch.bfloat16,
    bnb_8bit_quant_type="nf4",
    bnb_8bit_use_double_quant=True,
)


model = AutoModelForCausalLM.from_pretrained(
    model_checkpoint,
    trust_remote_code=True,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    quantization_config=bnb_config, # Use quantization_config instead of load_in_8bit
)

model.resize_token_embeddings(len(tokenizer))

# Prepare model for 8-bit training and add LoRA adapters
model = prepare_model_for_kbit_training(model)

lora_config = LoraConfig(
    r=16, # LoRA attention dimension
    lora_alpha=32, # Alpha parameter for LoRA scaling
    target_modules=["q_proj", "v_proj"], # Modules to apply LoRA to
    lora_dropout=0.05, # Dropout probability for LoRA layers
    bias="none", # Bias type
    task_type="CAUSAL_LM", # Task type
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()


# -----------------------------
# Step 4: Training setup
# -----------------------------

output_dir = "./deepseek_moe_expert_prompting"

training_args = TrainingArguments(
    output_dir=output_dir,
    per_device_train_batch_size=1, # Reduced batch size
    gradient_accumulation_steps=8, # Increased gradient accumulation steps
    num_train_epochs=1,
    save_steps=500,
    logging_dir="./logs",
    logging_steps=50,
    do_eval=False,
    report_to="none",
    seed=SEED,
    # Add checkpointing arguments
    save_strategy="steps",
    save_total_limit=2, # Keep up to 2 checkpoints
    load_best_model_at_end=False, # We are not doing evaluation
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_dataset,
    tokenizer=tokenizer,
    data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False),
)

print("Starting training...")

# Check if a checkpoint exists to resume from
latest_checkpoint = None
if os.path.exists(output_dir):
    # This assumes checkpoint directories are named like "checkpoint-XXX"
    checkpoint_dirs = [d for d in os.listdir(output_dir) if os.path.isdir(os.path.join(output_dir, d)) and d.startswith("checkpoint-")]
    if checkpoint_dirs:
        # Find the latest checkpoint based on step number
        latest_checkpoint_dir = max(checkpoint_dirs, key=lambda d: int(d.split("-")[1]))
        latest_checkpoint = os.path.join(output_dir, latest_checkpoint_dir)
        print(f"Resuming from checkpoint: {latest_checkpoint}")

# Use resume_from_checkpoint if a checkpoint is found
trainer.train(resume_from_checkpoint=latest_checkpoint)

print("Training complete.")

trainer.save_model(output_dir)
tokenizer.save_pretrained(output_dir)
```
config.json: 
 1.07k/? [00:00<00:00, 104kB/s]
configuration_deepseek.py: 
 10.2k/? [00:00<00:00, 1.23MB/s]
A new version of the following files was downloaded from https://huggingface.co/deepseek-ai/deepseek-moe-16b-base:
- configuration_deepseek.py
. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.
modeling_deepseek.py: 
 72.7k/? [00:00<00:00, 8.84MB/s]
A new version of the following files was downloaded from https://huggingface.co/deepseek-ai/deepseek-moe-16b-base:
- modeling_deepseek.py
. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.
model.safetensors.index.json: 
 490k/? [00:00<00:00, 27.5MB/s]
Fetching 7 files:   0%
 0/7 [00:10<?, ?it/s]
model-00001-of-00007.safetensors:  22%
 1.07G/5.00G [00:10<00:15, 245MB/s]
model-00005-of-00007.safetensors:  21%
 1.07G/5.00G [00:10<00:33, 116MB/s]
model-00006-of-00007.safetensors:  11%
 539M/5.00G [00:10<00:25, 178MB/s]
model-00004-of-00007.safetensors:  17%
 873M/5.00G [00:09<00:24, 169MB/s]
model-00007-of-00007.safetensors:  53%
 1.48G/2.77G [00:09<00:05, 223MB/s]
model-00002-of-00007.safetensors:   0%
 747k/5.00G [00:10<10:41:29, 130kB/s]
model-00003-of-00007.safetensors:  16%
 807M/5.00G [00:10<00:23, 175MB/s]
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[4], line 14
      1 # -----------------------------
      2 # Step 3: Load model and resize embeddings
      3 # -----------------------------
      4 
      5 # Configure 8-bit quantization
      6 bnb_config = BitsAndBytesConfig(
      7     load_in_8bit=True,
      8     bnb_8bit_compute_dtype=torch.bfloat16,
      9     bnb_8bit_quant_type="nf4",
     10     bnb_8bit_use_double_quant=True,
     11 )
---> 14 model = AutoModelForCausalLM.from_pretrained(
     15     model_checkpoint,
     16     trust_remote_code=True,
     17     torch_dtype=torch.bfloat16,
     18     device_map="auto",
     19     quantization_config=bnb_config, # Use quantization_config instead of load_in_8bit
     20 )
     22 model.resize_token_embeddings(len(tokenizer))
     24 # Prepare model for 8-bit training and add LoRA adapters

File /usr/local/lib/python3.10/dist-packages/transformers/models/auto/auto_factory.py:593, in _BaseAutoModelClass.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    591         model_class.register_for_auto_class(auto_class=cls)
    592     model_class = add_generation_mixin_to_remote_model(model_class)
--> 593     return model_class.from_pretrained(
    594         pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
    595     )
    596 elif type(config) in cls._model_mapping.keys():
    597     model_class = _get_model_class(config, cls._model_mapping)

File /usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:311, in restore_default_torch_dtype.<locals>._wrapper(*args, **kwargs)
    309 old_dtype = torch.get_default_dtype()
    310 try:
--> 311     return func(*args, **kwargs)
    312 finally:
    313     torch.set_default_dtype(old_dtype)

File /usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4680, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)
   4670 if (
   4671     gguf_file
   4672     and device_map is not None
   4673     and ((isinstance(device_map, dict) and "disk" in device_map.values()) or "disk" in device_map)
   4674 ):
   4675     raise RuntimeError(
   4676         "One or more modules is configured to be mapped to disk. Disk offload is not supported for models "
   4677         "loaded from GGUF files."
   4678     )
-> 4680 checkpoint_files, sharded_metadata = _get_resolved_checkpoint_files(
   4681     pretrained_model_name_or_path=pretrained_model_name_or_path,
   4682     subfolder=subfolder,
   4683     variant=variant,
   4684     gguf_file=gguf_file,
   4685     from_tf=from_tf,
   4686     from_flax=from_flax,
   4687     use_safetensors=use_safetensors,
   4688     cache_dir=cache_dir,
   4689     force_download=force_download,
   4690     proxies=proxies,
   4691     local_files_only=local_files_only,
   4692     token=token,
   4693     user_agent=user_agent,
   4694     revision=revision,
   4695     commit_hash=commit_hash,
   4696     is_remote_code=cls._auto_class is not None,
   4697     transformers_explicit_filename=transformers_explicit_filename,
   4698 )
   4700 is_sharded = sharded_metadata is not None
   4701 is_quantized = hf_quantizer is not None

File /usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:1295, in _get_resolved_checkpoint_files(pretrained_model_name_or_path, subfolder, variant, gguf_file, from_tf, from_flax, use_safetensors, cache_dir, force_download, proxies, local_files_only, token, user_agent, revision, commit_hash, is_remote_code, transformers_explicit_filename)
   1293 sharded_metadata = None
   1294 if is_sharded:
-> 1295     checkpoint_files, sharded_metadata = get_checkpoint_shard_files(
   1296         pretrained_model_name_or_path,
   1297         resolved_archive_file,
   1298         cache_dir=cache_dir,
   1299         force_download=force_download,
   1300         proxies=proxies,
   1301         local_files_only=local_files_only,
   1302         token=token,
   1303         user_agent=user_agent,
   1304         revision=revision,
   1305         subfolder=subfolder,
   1306         _commit_hash=commit_hash,
   1307     )
   1308 else:
   1309     checkpoint_files = [resolved_archive_file] if pretrained_model_name_or_path is not None else None

File /usr/local/lib/python3.10/dist-packages/transformers/utils/hub.py:1110, in get_checkpoint_shard_files(pretrained_model_name_or_path, index_filename, cache_dir, force_download, proxies, resume_download, local_files_only, token, user_agent, revision, subfolder, _commit_hash, **deprecated_kwargs)
   1106     return shard_filenames, sharded_metadata
   1108 # At this stage pretrained_model_name_or_path is a model identifier on the Hub. Try to get everything from cache,
   1109 # or download the files
-> 1110 cached_filenames = cached_files(
   1111     pretrained_model_name_or_path,
   1112     shard_filenames,
   1113     cache_dir=cache_dir,
   1114     force_download=force_download,
   1115     proxies=proxies,
   1116     resume_download=resume_download,
   1117     local_files_only=local_files_only,
   1118     token=token,
   1119     user_agent=user_agent,
   1120     revision=revision,
   1121     subfolder=subfolder,
   1122     _commit_hash=_commit_hash,
   1123 )
   1125 return cached_filenames, sharded_metadata

File /usr/local/lib/python3.10/dist-packages/transformers/utils/hub.py:557, in cached_files(path_or_repo_id, filenames, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)
    554     # Any other Exception type should now be re-raised, in order to provide helpful error messages and break the execution flow
    555     # (EntryNotFoundError will be treated outside this block and correctly re-raised if needed)
    556     elif not isinstance(e, EntryNotFoundError):
--> 557         raise e
    559 resolved_files = [
    560     _get_cache_file_to_return(path_or_repo_id, filename, cache_dir, revision) for filename in full_filenames
    561 ]
    562 # If there are any missing file and the flag is active, raise

File /usr/local/lib/python3.10/dist-packages/transformers/utils/hub.py:485, in cached_files(path_or_repo_id, filenames, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)
    470         hf_hub_download(
    471             path_or_repo_id,
    472             filenames[0],
   (...)
    482             local_files_only=local_files_only,
    483         )
    484     else:
--> 485         snapshot_download(
    486             path_or_repo_id,
    487             allow_patterns=full_filenames,
    488             repo_type=repo_type,
    489             revision=revision,
    490             cache_dir=cache_dir,
    491             user_agent=user_agent,
    492             force_download=force_download,
    493             proxies=proxies,
    494             resume_download=resume_download,
    495             token=token,
    496             local_files_only=local_files_only,
    497         )
    499 except Exception as e:
    500     # We cannot recover from them
    501     if isinstance(e, RepositoryNotFoundError) and not isinstance(e, GatedRepoError):

File /usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_validators.py:114, in validate_hf_hub_args.<locals>._inner_fn(*args, **kwargs)
    111 if check_use_auth_token:
    112     kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_token, kwargs=kwargs)
--> 114 return fn(*args, **kwargs)

File /usr/local/lib/python3.10/dist-packages/huggingface_hub/_snapshot_download.py:327, in snapshot_download(repo_id, repo_type, revision, cache_dir, local_dir, library_name, library_version, user_agent, proxies, etag_timeout, force_download, token, local_files_only, allow_patterns, ignore_patterns, max_workers, tqdm_class, headers, endpoint, local_dir_use_symlinks, resume_download)
    325         _inner_hf_hub_download(file)
    326 else:
--> 327     thread_map(
    328         _inner_hf_hub_download,
    329         filtered_repo_files,
    330         desc=tqdm_desc,
    331         max_workers=max_workers,
    332         # User can use its own tqdm class or the default one from `huggingface_hub.utils`
    333         tqdm_class=tqdm_class or hf_tqdm,
    334     )
    336 if local_dir is not None:
    337     return str(os.path.realpath(local_dir))

File /usr/local/lib/python3.10/dist-packages/tqdm/contrib/concurrent.py:69, in thread_map(fn, *iterables, **tqdm_kwargs)
     55 """
     56 Equivalent of `list(map(fn, *iterables))`
     57 driven by `concurrent.futures.ThreadPoolExecutor`.
   (...)
     66     [default: max(32, cpu_count() + 4)].
     67 """
     68 from concurrent.futures import ThreadPoolExecutor
---> 69 return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs)

File /usr/local/lib/python3.10/dist-packages/tqdm/contrib/concurrent.py:51, in _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs)
     47 with ensure_lock(tqdm_class, lock_name=lock_name) as lk:
     48     # share lock in case workers are already using `tqdm`
     49     with PoolExecutor(max_workers=max_workers, initializer=tqdm_class.set_lock,
     50                       initargs=(lk,)) as ex:
---> 51         return list(tqdm_class(ex.map(fn, *iterables, chunksize=chunksize), **kwargs))

File /usr/local/lib/python3.10/dist-packages/tqdm/notebook.py:250, in tqdm_notebook.__iter__(self)
    248 try:
    249     it = super().__iter__()
--> 250     for obj in it:
    251         # return super(tqdm...) will not catch exception
    252         yield obj
    253 # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt

File /usr/local/lib/python3.10/dist-packages/tqdm/std.py:1181, in tqdm.__iter__(self)
   1178 time = self._time
   1180 try:
-> 1181     for obj in iterable:
   1182         yield obj
   1183         # Update and possibly print the progressbar.
   1184         # Note: does not call self.update(1) for speed optimisation.

File /usr/lib/python3.10/concurrent/futures/_base.py:621, in Executor.map.<locals>.result_iterator()
    618 while fs:
    619     # Careful not to keep a reference to the popped future
    620     if timeout is None:
--> 621         yield _result_or_cancel(fs.pop())
    622     else:
    623         yield _result_or_cancel(fs.pop(), end_time - time.monotonic())

File /usr/lib/python3.10/concurrent/futures/_base.py:319, in _result_or_cancel(***failed resolving arguments***)
    317 try:
    318     try:
--> 319         return fut.result(timeout)
    320     finally:
    321         fut.cancel()

File /usr/lib/python3.10/concurrent/futures/_base.py:458, in Future.result(self, timeout)
    456     raise CancelledError()
    457 elif self._state == FINISHED:
--> 458     return self.__get_result()
    459 else:
    460     raise TimeoutError()

File /usr/lib/python3.10/concurrent/futures/_base.py:403, in Future.__get_result(self)
    401 if self._exception:
    402     try:
--> 403         raise self._exception
    404     finally:
    405         # Break a reference cycle with the exception in self._exception
    406         self = None

File /usr/lib/python3.10/concurrent/futures/thread.py:58, in _WorkItem.run(self)
     55     return
     57 try:
---> 58     result = self.fn(*self.args, **self.kwargs)
     59 except BaseException as exc:
     60     self.future.set_exception(exc)

File /usr/local/lib/python3.10/dist-packages/huggingface_hub/_snapshot_download.py:301, in snapshot_download.<locals>._inner_hf_hub_download(repo_file)
    300 def _inner_hf_hub_download(repo_file: str):
--> 301     return hf_hub_download(
    302         repo_id,
    303         filename=repo_file,
    304         repo_type=repo_type,
    305         revision=commit_hash,
    306         endpoint=endpoint,
    307         cache_dir=cache_dir,
    308         local_dir=local_dir,
    309         local_dir_use_symlinks=local_dir_use_symlinks,
    310         library_name=library_name,
    311         library_version=library_version,
    312         user_agent=user_agent,
    313         proxies=proxies,
    314         etag_timeout=etag_timeout,
    315         resume_download=resume_download,
    316         force_download=force_download,
    317         token=token,
    318         headers=headers,
    319     )

File /usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_validators.py:114, in validate_hf_hub_args.<locals>._inner_fn(*args, **kwargs)
    111 if check_use_auth_token:
    112     kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_token, kwargs=kwargs)
--> 114 return fn(*args, **kwargs)

File /usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1008, in hf_hub_download(repo_id, filename, subfolder, repo_type, revision, library_name, library_version, cache_dir, local_dir, user_agent, force_download, proxies, etag_timeout, token, local_files_only, headers, endpoint, resume_download, force_filename, local_dir_use_symlinks)
    988     return _hf_hub_download_to_local_dir(
    989         # Destination
    990         local_dir=local_dir,
   (...)
   1005         local_files_only=local_files_only,
   1006     )
   1007 else:
-> 1008     return _hf_hub_download_to_cache_dir(
   1009         # Destination
   1010         cache_dir=cache_dir,
   1011         # File info
   1012         repo_id=repo_id,
   1013         filename=filename,
   1014         repo_type=repo_type,
   1015         revision=revision,
   1016         # HTTP info
   1017         endpoint=endpoint,
   1018         etag_timeout=etag_timeout,
   1019         headers=hf_headers,
   1020         proxies=proxies,
   1021         token=token,
   1022         # Additional options
   1023         local_files_only=local_files_only,
   1024         force_download=force_download,
   1025     )

File /usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1161, in _hf_hub_download_to_cache_dir(cache_dir, repo_id, filename, repo_type, revision, endpoint, etag_timeout, headers, proxies, token, local_files_only, force_download)
   1158 # Local file doesn't exist or etag isn't a match => retrieve file from remote (or cache)
   1160 with WeakFileLock(lock_path):
-> 1161     _download_to_tmp_and_move(
   1162         incomplete_path=Path(blob_path + ".incomplete"),
   1163         destination_path=Path(blob_path),
   1164         url_to_download=url_to_download,
   1165         proxies=proxies,
   1166         headers=headers,
   1167         expected_size=expected_size,
   1168         filename=filename,
   1169         force_download=force_download,
   1170         etag=etag,
   1171         xet_file_data=xet_file_data,
   1172     )
   1173     if not os.path.exists(pointer_path):
   1174         _create_symlink(blob_path, pointer_path, new_blob=True)

File /usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1710, in _download_to_tmp_and_move(incomplete_path, destination_path, url_to_download, proxies, headers, expected_size, filename, force_download, etag, xet_file_data)
   1708 if xet_file_data is not None and is_xet_available():
   1709     logger.debug("Xet Storage is enabled for this repo. Downloading file from Xet Storage..")
-> 1710     xet_get(
   1711         incomplete_path=incomplete_path,
   1712         xet_file_data=xet_file_data,
   1713         headers=headers,
   1714         expected_size=expected_size,
   1715         displayed_filename=filename,
   1716     )
   1717 else:
   1718     if xet_file_data is not None:

File /usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:627, in xet_get(incomplete_path, xet_file_data, headers, expected_size, displayed_filename, _tqdm_bar)
    624 def progress_updater(progress_bytes: float):
    625     progress.update(progress_bytes)
--> 627 download_files(
    628     xet_download_info,
    629     endpoint=connection_info.endpoint,
    630     token_info=(connection_info.access_token, connection_info.expiration_unix_epoch),
    631     token_refresher=token_refresher,
    632     progress_updater=[progress_updater],
    633 )

RuntimeError: Data processing error: CAS service error : IO Error: No space left on device (os error 28)

Additionally, I keep getting this popup:
File Save Error for deepseek-moe-16b.ipynb
Unexpected error while saving file: workspace/deepseek-moe-16b.ipynb unable to open database file

Context: This code was made for a colab notebook. Its now being run on jupyter notebook via runpod.