Repository: oumi-ai/oumi
File: src/oumi/core/processors/default_processor.py (Python)

Code before:
        ):
            raise ValueError(
                "Worker processor doesn't have the `apply_chat_template` method"
            )

        self._processor_name = processor_name
        self._worker_processor: Callable = worker_processor
        self._worker_processor.tokenizer = tokenizer
        self._tokenizer: BaseTokenizer = tokenizer

        # If the worker processor has a different chat template, then update it.
        # This also prevents runtime errors when the worker's processor template
        # is a read-only attribute and identical to tokenizer's template.
        if self._worker_processor.chat_template != tokenizer.chat_template:
            self._worker_processor.chat_template = tokenizer.chat_template

        self._image_processor: Optional[BaseImageProcessor] = None
        if (
            hasattr(self._worker_processor, "image_processor")
            and self._worker_processor.image_processor is not None
        ):
            self._image_processor = DefaultImageProcessor(
                self._worker_processor.image_processor
            )
        self._label_ignore_index: Optional[int] = label_ignore_index
        self._ignore_features: Optional[list[str]] = ignore_features

    @property
    @override
    def processor_name(self) -> str:
        """Returns a processor name."""
        return self._processor_name

    @property
    @override
    def tokenizer(self) -> BaseTokenizer:
        """Returns a tokenizer associated with this processor."""
        return self._tokenizer

    @tokenizer.setter
    @override
    def tokenizer(self, new_tokenizer: BaseTokenizer) -> None:
        """Sets a tokenizer associated with this processor."""
        self._worker_processor.tokenizer = new_tokenizer
        self._tokenizer = new_tokenizer

    @property
    @override
    def chat_template(self) -> str:
        """Returns a chat template."""
        if not hasattr(self._worker_processor, "chat_template"):

Review comment (suggestion):
Consider making our own private copy. Just to make sure that external users can't mutate our member variable.
```
self._ignore_features: Optional[list[str]] =  copy.copy(ignore_features) if ignore_features else [] 
```

Code after:
            and callable(worker_processor.apply_chat_template)
        ):
            raise ValueError(
                "Worker processor doesn't have the `apply_chat_template` method"
            )

        self._processor_name = processor_name
        self._worker_processor: Callable = worker_processor
        self._worker_processor.tokenizer = tokenizer
        self._tokenizer: BaseTokenizer = tokenizer

        # If the worker processor has a different chat template, then update it.
        # This also prevents runtime errors when the worker's processor template
        # is a read-only attribute and identical to tokenizer's template.
        if self._worker_processor.chat_template != tokenizer.chat_template:
            self._worker_processor.chat_template = tokenizer.chat_template

        self._image_processor: Optional[BaseImageProcessor] = None
        if (
            hasattr(self._worker_processor, "image_processor")
            and self._worker_processor.image_processor is not None
        ):
            self._image_processor = DefaultImageProcessor(
                self._worker_processor.image_processor
            )
        self._label_ignore_index: Optional[int] = label_ignore_index
        self._ignore_features: Optional[list[str]] = (
            copy.copy(ignore_features) if ignore_features else []
        )

    @property
    @override
    def processor_name(self) -> str:
        """Returns a processor name."""
        return self._processor_name

    @property
    @override
    def tokenizer(self) -> BaseTokenizer:
        """Returns a tokenizer associated with this processor."""
        return self._tokenizer

    @tokenizer.setter
    @override
    def tokenizer(self, new_tokenizer: BaseTokenizer) -> None:
        """Sets a tokenizer associated with this processor."""
        self._worker_processor.tokenizer = new_tokenizer
        self._tokenizer = new_tokenizer

    @property
    @override