Repository: huggingface/transformers
File: tests/models/owlvit/test_modeling_owlvit.py (Python)

Code before:
            outputs.logits_per_image.shape,
            torch.Size((inputs.pixel_values.shape[0], inputs.input_ids.shape[0])),
        )
        self.assertEqual(
            outputs.logits_per_text.shape,
            torch.Size((inputs.input_ids.shape[0], inputs.pixel_values.shape[0])),
        )
        expected_logits = torch.tensor([[3.6278, 0.8861]], device=torch_device)
        torch.testing.assert_close(outputs.logits_per_image, expected_logits, rtol=1e-3, atol=1e-3)

        expected_shape = torch.Size((1, 626, 768))
        self.assertEqual(outputs.vision_model_output.last_hidden_state.shape, expected_shape)

        # OwlViTForObjectDetection part.
        model = OwlViTForObjectDetection.from_pretrained(model_name).to(torch_device)

        with torch.no_grad():
            outputs = model(**inputs, interpolate_pos_encoding=True)

        num_queries = int((inputs.pixel_values.shape[-1] // model.config.vision_config.patch_size) ** 2)
        self.assertEqual(outputs.pred_boxes.shape, torch.Size((1, num_queries, 4)))

        expected_slice_boxes = torch.tensor(
            [[0.0680, 0.0422, 0.1347], [0.2071, 0.0450, 0.4146], [0.2000, 0.0418, 0.3476]]
        ).to(torch_device)
        torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_slice_boxes, rtol=1e-2, atol=1e-2)

        model = OwlViTForObjectDetection.from_pretrained(model_name).to(torch_device)
        query_image = prepare_img()
        inputs = processor(
            images=image,
            query_images=query_image,
            max_length=16,
            padding="max_length",
            return_tensors="pt",
        ).to(torch_device)

        with torch.no_grad():
            outputs = model.image_guided_detection(**inputs, interpolate_pos_encoding=True)

        # No need to check the logits, we just check inference runs fine.
        num_queries = int((inputs.pixel_values.shape[-1] / model.config.vision_config.patch_size) ** 2)
        self.assertEqual(outputs.target_pred_boxes.shape, torch.Size((1, num_queries, 4)))

        # Deactivate interpolate_pos_encoding on same model, and use default image size.
        # Verify the dynamic change caused by the activation/deactivation of interpolate_pos_encoding of variables: (self.sqrt_num_patch_h, self.sqrt_num_patch_w), self.box_bias from (OwlViTForObjectDetection).
        processor = OwlViTProcessor.from_pretrained(model_name)

        image = prepare_img()
        inputs = processor(
            text=[["a photo of a cat", "a photo of a dog"]],

Review comment (suggestion):
The tolerance is a bit too high, do we really 
- need to change that many
- have this high tol, i.e. is 1e-3 maybe enough instead

Code after:
            outputs.logits_per_image.shape,
            torch.Size((inputs.pixel_values.shape[0], inputs.input_ids.shape[0])),
        )
        self.assertEqual(
            outputs.logits_per_text.shape,
            torch.Size((inputs.input_ids.shape[0], inputs.pixel_values.shape[0])),
        )
        expected_logits = torch.tensor([[3.6282, 0.8859]], device=torch_device)
        torch.testing.assert_close(outputs.logits_per_image, expected_logits, rtol=1e-3, atol=1e-3)

        expected_shape = torch.Size((1, 626, 768))
        self.assertEqual(outputs.vision_model_output.last_hidden_state.shape, expected_shape)

        # OwlViTForObjectDetection part.
        model = OwlViTForObjectDetection.from_pretrained(model_name).to(torch_device)

        with torch.no_grad():
            outputs = model(**inputs, interpolate_pos_encoding=True)

        num_queries = int((inputs.pixel_values.shape[-1] // model.config.vision_config.patch_size) ** 2)
        self.assertEqual(outputs.pred_boxes.shape, torch.Size((1, num_queries, 4)))

        expected_slice_boxes = torch.tensor(
            [[0.0679, 0.0422, 0.1347], [0.2073, 0.0450, 0.4151], [0.2002, 0.0418, 0.3483]]
        ).to(torch_device)
        torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_slice_boxes, rtol=1e-4, atol=1e-4)

        model = OwlViTForObjectDetection.from_pretrained(model_name).to(torch_device)
        query_image = prepare_img()
        inputs = processor(
            images=image,
            query_images=query_image,
            max_length=16,
            padding="max_length",
            return_tensors="pt",
        ).to(torch_device)

        with torch.no_grad():
            outputs = model.image_guided_detection(**inputs, interpolate_pos_encoding=True)

        # No need to check the logits, we just check inference runs fine.
        num_queries = int((inputs.pixel_values.shape[-1] / model.config.vision_config.patch_size) ** 2)
        self.assertEqual(outputs.target_pred_boxes.shape, torch.Size((1, num_queries, 4)))

        # Deactivate interpolate_pos_encoding on same model, and use default image size.
        # Verify the dynamic change caused by the activation/deactivation of interpolate_pos_encoding of variables: (self.sqrt_num_patch_h, self.sqrt_num_patch_w), self.box_bias from (OwlViTForObjectDetection).
        processor = OwlViTProcessor.from_pretrained(model_name)

        image = prepare_img()
        inputs = processor(
            text=[["a photo of a cat", "a photo of a dog"]],