Repository: huggingface/transformers
File: tests/test_processing_common.py (Python)

Code before:

        image_processor = self.get_component("image_processor")
        tokenizer = self.get_component("tokenizer")
        if not tokenizer.pad_token:
            tokenizer.pad_token = "[TEST_PAD]"
        processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
        self.skip_processor_without_typed_kwargs(processor)
        input_str = "lower newer"
        image_input = self.prepare_image_inputs()

        # Define the kwargs for each modality
        all_kwargs = {
            "common_kwargs": {"return_tensors": "pt"},
            "images_kwargs": {"crop_size": {"height": 214, "width": 214}, "size": {"height": 214, "width": 214}},
            "text_kwargs": {"padding": "max_length", "max_length": 76},
        }

        inputs = processor(text=input_str, images=image_input, **all_kwargs)
        self.assertEqual(inputs["pixel_values"].shape[2], 214)

        self.assertEqual(len(inputs["input_ids"][0]), 76)

    #  text + audio kwargs testing
    @require_torch
    def test_tokenizer_defaults_preserved_by_kwargs_audio(self):
        if "feature_extractor" not in self.processor_class.attributes:
            self.skipTest(f"feature_extractor attribute not present in {self.processor_class}")
        feature_extractor = self.get_component("feature_extractor")
        if hasattr(self, "get_tokenizer"):
            tokenizer = self.get_tokenizer(max_length=117, padding="max_length")
        elif hasattr(self, "get_component"):
            tokenizer = self.get_component("tokenizer", max_length=117, padding="max_length")
        if not tokenizer.pad_token:
            tokenizer.pad_token = "[TEST_PAD]"
        processor = self.processor_class(tokenizer=tokenizer, feature_extractor=feature_extractor)
        self.skip_processor_without_typed_kwargs(processor)
        input_str = "lower newer"
        raw_speech = floats_list((3, 1000))
        inputs = processor(text=input_str, audio=raw_speech, return_tensors="pt")
        if "input_ids" in inputs:
            self.assertEqual(len(inputs["input_ids"][0]), 117)
        elif "labels" in inputs:
            self.assertEqual(len(inputs["labels"][0]), 117)

    @require_torch
    def test_kwargs_overrides_default_tokenizer_kwargs_audio(self):
        if "feature_extractor" not in self.processor_class.attributes:
            self.skipTest(f"feature_extractor attribute not present in {self.processor_class}")
        feature_extractor = self.get_component("feature_extractor")
        if hasattr(self, "get_tokenizer"):
            tokenizer = self.get_tokenizer(max_length=117)

Review comment (suggestion):
Audio tests are not used by this PR, should be removed. Let us add them to the PR which updates audio processors ;)

Code after:
        super().__init__(image_processor, tokenizer)

        self.processor_attr_1 = processor_attr_1
        self.processor_attr_2 = processor_attr_2


@require_tokenizers
@require_vision
class ProcessorTest(unittest.TestCase):
    processor_class = MyProcessor

    def prepare_processor_dict(self):
        return {"processor_attr_1": 1, "processor_attr_2": False}

    def get_processor(self):
        image_processor = CLIPImageProcessor.from_pretrained("openai/clip-vit-large-patch14")
        tokenizer = CLIPTokenizerFast.from_pretrained("openai/clip-vit-large-patch14")
        processor = MyProcessor(image_processor, tokenizer, **self.prepare_processor_dict())

        return processor

    def test_processor_to_json_string(self):
        processor = self.get_processor()
        obj = json.loads(processor.to_json_string())
        for key, value in self.prepare_processor_dict().items():
            self.assertEqual(obj[key], value)
            self.assertEqual(getattr(processor, key, None), value)

    def test_processor_from_and_save_pretrained(self):
        processor_first = self.get_processor()

        with tempfile.TemporaryDirectory() as tmpdirname:
            saved_file = processor_first.save_pretrained(tmpdirname)[0]
            check_json_file_has_correct_format(saved_file)
            processor_second = self.processor_class.from_pretrained(tmpdirname)

        self.assertEqual(processor_second.to_dict(), processor_first.to_dict())