Problem with the masked language modeling tutorial 
Author: PatrickLJHarrell
Pull request: no

- `transformers` version: 4.41.2
- Platform: Windows-10-10.0.26120-SP0
- Python version: 3.11.9
- Huggingface_hub version: 0.23.4
- Safetensors version: 0.4.3
- Accelerate version: 0.31.0
- Accelerate config:    not found
- PyTorch version (GPU?): 2.3.0+cu121 (True)
- Tensorflow version (GPU?): not installed (NA)
- Flax version (CPU?/GPU?/TPU?): not installed (NA)
- Jax version: not installed
- JaxLib version: not installed
- Using GPU in script?: <fill in>
- Using distributed or parallel set-up in script?: <fill in>

I keep getting an error when following the masked language model tutorial using the following code as instructed.

from huggingface_hub import notebook_login
notebook_login()

from datasets import load_dataset

eli5 = load_dataset("eli5_category", split="train[:5000]")

eli5["train"][0]

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert/distilroberta-base")


eli5 = eli5.flatten()
eli5["train"][0]

def preprocess_function(examples):
    return tokenizer([" ".join(x) for x in examples["answers.text"]])

tokenized_eli5 = eli5.map(
    preprocess_function,
    batched=True,
    num_proc=4,
    remove_columns=eli5["train"].column_names,
)


Error: 

{
	"name": "NameError",
	"message": "name 'tokenizer' is not defined",
	"stack": "---------------------------------------------------------------------------
RemoteTraceback                           Traceback (most recent call last)
RemoteTraceback: 
\"\"\"
Traceback (most recent call last):
  File \"c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\multiprocess\\pool.py\", line 125, in worker
    result = (True, func(*args, **kwds))
                    ^^^^^^^^^^^^^^^^^^^
  File \"c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\utils\\py_utils.py\", line 678, in _write_generator_to_queue
    for i, result in enumerate(func(**kwargs)):
  File \"c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\arrow_dataset.py\", line 3552, in _map_single
    batch = apply_function_on_filtered_inputs(
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File \"c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\arrow_dataset.py\", line 3421, in apply_function_on_filtered_inputs
    processed_inputs = function(*fn_args, *additional_args, **fn_kwargs)
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File \"C:\\Users\\ljjx\\AppData\\Local\\Temp\\ipykernel_36716\\2763327562.py\", line 2, in preprocess_function
NameError: name 'tokenizer' is not defined
\"\"\"

The above exception was the direct cause of the following exception:

NameError                                 Traceback (most recent call last)
Cell In[10], line 1
----> 1 tokenized_eli5 = eli5.map(
      2     preprocess_function,
      3     batched=True,
      4     num_proc=4,
      5     remove_columns=eli5[\"train\"].column_names,
      6 )

File c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\dataset_dict.py:869, in DatasetDict.map(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_names, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, desc)
    866 if cache_file_names is None:
    867     cache_file_names = {k: None for k in self}
    868 return DatasetDict(
--> 869     {
    870         k: dataset.map(
    871             function=function,
    872             with_indices=with_indices,
    873             with_rank=with_rank,
    874             input_columns=input_columns,
    875             batched=batched,
    876             batch_size=batch_size,
    877             drop_last_batch=drop_last_batch,
    878             remove_columns=remove_columns,
    879             keep_in_memory=keep_in_memory,
    880             load_from_cache_file=load_from_cache_file,
    881             cache_file_name=cache_file_names[k],
    882             writer_batch_size=writer_batch_size,
    883             features=features,
    884             disable_nullable=disable_nullable,
    885             fn_kwargs=fn_kwargs,
    886             num_proc=num_proc,
    887             desc=desc,
    888         )
    889         for k, dataset in self.items()
    890     }
    891 )

File c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\dataset_dict.py:870, in <dictcomp>(.0)
    866 if cache_file_names is None:
    867     cache_file_names = {k: None for k in self}
    868 return DatasetDict(
    869     {
--> 870         k: dataset.map(
    871             function=function,
    872             with_indices=with_indices,
    873             with_rank=with_rank,
    874             input_columns=input_columns,
    875             batched=batched,
    876             batch_size=batch_size,
    877             drop_last_batch=drop_last_batch,
    878             remove_columns=remove_columns,
    879             keep_in_memory=keep_in_memory,
    880             load_from_cache_file=load_from_cache_file,
    881             cache_file_name=cache_file_names[k],
    882             writer_batch_size=writer_batch_size,
    883             features=features,
    884             disable_nullable=disable_nullable,
    885             fn_kwargs=fn_kwargs,
    886             num_proc=num_proc,
    887             desc=desc,
    888         )
    889         for k, dataset in self.items()
    890     }
    891 )

File c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\arrow_dataset.py:602, in transmit_tasks.<locals>.wrapper(*args, **kwargs)
    600     self: \"Dataset\" = kwargs.pop(\"self\")
    601 # apply actual function
--> 602 out: Union[\"Dataset\", \"DatasetDict\"] = func(self, *args, **kwargs)
    603 datasets: List[\"Dataset\"] = list(out.values()) if isinstance(out, dict) else [out]
    604 for dataset in datasets:
    605     # Remove task templates if a column mapping of the template is no longer valid

File c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\arrow_dataset.py:567, in transmit_format.<locals>.wrapper(*args, **kwargs)
    560 self_format = {
    561     \"type\": self._format_type,
    562     \"format_kwargs\": self._format_kwargs,
    563     \"columns\": self._format_columns,
    564     \"output_all_columns\": self._output_all_columns,
    565 }
    566 # apply actual function
--> 567 out: Union[\"Dataset\", \"DatasetDict\"] = func(self, *args, **kwargs)
    568 datasets: List[\"Dataset\"] = list(out.values()) if isinstance(out, dict) else [out]
    569 # re-apply format to the output

File c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\arrow_dataset.py:3253, in Dataset.map(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint, desc)
   3247 logger.info(f\"Spawning {num_proc} processes\")
   3248 with hf_tqdm(
   3249     unit=\" examples\",
   3250     total=pbar_total,
   3251     desc=(desc or \"Map\") + f\" (num_proc={num_proc})\",
   3252 ) as pbar:
-> 3253     for rank, done, content in iflatmap_unordered(
   3254         pool, Dataset._map_single, kwargs_iterable=kwargs_per_job
   3255     ):
   3256         if done:
   3257             shards_done += 1

File c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\utils\\py_utils.py:718, in iflatmap_unordered(pool, func, kwargs_iterable)
    715 finally:
    716     if not pool_changed:
    717         # we get the result in case there's an error to raise
--> 718         [async_result.get(timeout=0.05) for async_result in async_results]

File c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\datasets\\utils\\py_utils.py:718, in <listcomp>(.0)
    715 finally:
    716     if not pool_changed:
    717         # we get the result in case there's an error to raise
--> 718         [async_result.get(timeout=0.05) for async_result in async_results]

File c:\\Users\\ljjx\\HFModels\\.venv\\Lib\\site-packages\\multiprocess\\pool.py:774, in ApplyResult.get(self, timeout)
    772     return self._value
    773 else:
--> 774     raise self._value

NameError: name 'tokenizer' is not defined"
}

But when I define a tokenizer it says map() doesn't take a tokenizer.

Am I doing something wrong here? I've tried typing everything myself and also copying the code directly from the page.

https://huggingface.co/docs/transformers/en/tasks/masked_language_modeling

Comments:
I think I solved it I had to import the tokenizer inside of the preprocess_function()
"def preprocess_function(examples):
    # Import AutoTokenizer inside the function to ensure it's recognized in multiprocessing context
    from transformers import AutoTokenizer
    tokenizer = AutoTokenizer.from_pretrained("distilroberta-base")
    return tokenizer([" ".join(x) for x in examples["questions.text"]])"

Odd, but it worked. 

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) are likely to be ignored.