Repository: huggingface/transformers
Pull request: skip `test_torchscript_*` for now until the majority of the community ask for it

# What does this PR do?

As discussed with the core maintainer(s) offline.

For the record, the trace log (after #39120) is


```bash
>                       traced_model = torch.jit.trace(model, (main_input,))

tests/test_modeling_common.py:1453:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py:838: in _fn
    return fn(*args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/jit/_trace.py:1002: in trace
    traced_func = _trace_impl(
/usr/local/lib/python3.10/dist-packages/torch/jit/_trace.py:696: in _trace_impl
    return trace_module(
/usr/local/lib/python3.10/dist-packages/torch/jit/_trace.py:1279: in trace_module
    module._c._create_method_from_trace(
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1751: in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1762: in _call_impl
    return forward_call(*args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1741: in _slow_forward
    result = self.forward(*input, **kwargs)
src/transformers/utils/generic.py:1060: in wrapper
    outputs = func(self, *args, **kwargs)
src/transformers/models/llama/modeling_llama.py:396: in forward
    causal_mask = create_causal_mask(
src/transformers/masking_utils.py:793: in create_causal_mask
    causal_mask = mask_interface(
src/transformers/masking_utils.py:482: in eager_mask
    mask = sdpa_mask(
src/transformers/masking_utils.py:366: in sdpa_mask_recent_torch
    causal_mask = _vmap_for_bhqkv(mask_function)(batch_arange, head_arange, cache_position, kv_arange)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/apis.py:202: in wrapped
    return vmap_impl(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/vmap.py:334: in vmap_impl
    return _flat_vmap(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/vmap.py:484: in _flat_vmap
    batched_outputs = func(*batched_inputs, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/apis.py:202: in wrapped
    return vmap_impl(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/vmap.py:334: in vmap_impl
    return _flat_vmap(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/vmap.py:484: in _flat_vmap
    batched_outputs = func(*batched_inputs, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/apis.py:202: in wrapped
    return vmap_impl(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/vmap.py:334: in vmap_impl
    return _flat_vmap(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/vmap.py:484: in _flat_vmap
    batched_outputs = func(*batched_inputs, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/apis.py:202: in wrapped
    return vmap_impl(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/vmap.py:334: in vmap_impl
    return _flat_vmap(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/vmap.py:484: in _flat_vmap
    batched_outputs = func(*batched_inputs, **kwargs)
src/transformers/masking_utils.py:49: in and_mask
    result = result & mask(batch_idx, head_idx, q_idx, kv_idx)
src/transformers/masking_utils.py:134: in inner_mask
    return packed_sequence_mask[batch_idx, q_idx] == packed_sequence_mask[batch_idx, kv_idx]
/usr/local/lib/python3.10/dist-packages/torch/_dynamo/_trace_wrapped_higher_order_op.py:141: in __torch_function__
    return mod_index(args[0], index_args)
/usr/local/lib/python3.10/dist-packages/torch/autograd/function.py:585: in apply
    return custom_function_call(cls, *args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:49: in __call__
    return super().__call__(autograd_function, *args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:471: in __call__
    return wrapper()
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:467: in wrapper
    return self.dispatch(
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:330: in dispatch
    return dispatch_functorch(self, args, kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/pyfunctorch.py:294: in dispatch_functorch
    return interpreter.process(op, args, kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/pyfunctorch.py:130: in process
    return kernel(self, *args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:300: in custom_function_call_vmap
    return custom_function_call_vmap_generate_rule(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:384: in custom_function_call_vmap_generate_rule
    outputs = custom_function_call(vmapped_function, *unwrapped_operands)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:49: in __call__
    return super().__call__(autograd_function, *args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:471: in __call__
    return wrapper()
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:467: in wrapper
    return self.dispatch(
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:330: in dispatch
    return dispatch_functorch(self, args, kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/pyfunctorch.py:294: in dispatch_functorch
    return interpreter.process(op, args, kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/pyfunctorch.py:130: in process
    return kernel(self, *args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:300: in custom_function_call_vmap
    return custom_function_call_vmap_generate_rule(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:384: in custom_function_call_vmap_generate_rule
    outputs = custom_function_call(vmapped_function, *unwrapped_operands)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:49: in __call__
    return super().__call__(autograd_function, *args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:471: in __call__
    return wrapper()
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:467: in wrapper
    return self.dispatch(
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:330: in dispatch
    return dispatch_functorch(self, args, kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/pyfunctorch.py:294: in dispatch_functorch
    return interpreter.process(op, args, kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/pyfunctorch.py:130: in process
    return kernel(self, *args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:300: in custom_function_call_vmap
    return custom_function_call_vmap_generate_rule(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:384: in custom_function_call_vmap_generate_rule
    outputs = custom_function_call(vmapped_function, *unwrapped_operands)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:49: in __call__
    return super().__call__(autograd_function, *args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:471: in __call__
    return wrapper()
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:467: in wrapper
    return self.dispatch(
/usr/local/lib/python3.10/dist-packages/torch/_ops.py:330: in dispatch
    return dispatch_functorch(self, args, kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/pyfunctorch.py:294: in dispatch_functorch
    return interpreter.process(op, args, kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/pyfunctorch.py:130: in process
    return kernel(self, *args, **kwargs)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:300: in custom_function_call_vmap
    return custom_function_call_vmap_generate_rule(
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:384: in custom_function_call_vmap_generate_rule
    outputs = custom_function_call(vmapped_function, *unwrapped_operands)
/usr/local/lib/python3.10/dist-packages/torch/_functorch/autograd_function.py:50: in __call__
    return autograd_function.apply(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

cls = <class 'torch._functorch.autograd_function.VmappedVmappedVmappedVmappedModIndex'>
args = (tensor([[0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, ... 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12], device='cuda:0'), tensor([0, 1, 2, 3, 4, 5, 6], device='cuda:0')]), kwargs = {}
bind_default_args = <function Function.apply.<locals>.bind_default_args at 0x7fda91408700>, is_setup_ctx_defined = True

    @classmethod
    def apply(cls, *args, **kwargs):
        def bind_default_args(func, *args, **kwargs):
            signature = inspect.signature(func)
            bound_args = signature.bind(*args, **kwargs)
            bound_args.apply_defaults()

            return bound_args.args

        is_setup_ctx_defined = _is_setup_context_defined(cls.setup_context)
        if is_setup_ctx_defined:
            args = bind_default_args(cls.forward, *args, **kwargs)

        if not torch._C._are_functorch_transforms_active():
            # See NOTE: [functorch vjp and autograd interaction]
            args = _functorch.utils.unwrap_dead_wrappers(args)
>           return super().apply(*args, **kwargs)  # type: ignore[misc]
E           RuntimeError: _Map_base::at

/usr/local/lib/python3.10/dist-packages/torch/autograd/function.py:575: RuntimeError

During handling of the above exception, another exception occurred:

self = <tests.models.llama.test_modeling_llama.LlamaModelTest testMethod=test_torchscript_simple>

    @slow
    def test_torchscript_simple(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
>       self._create_and_check_torchscript(config, inputs_dict)

tests/test_modeling_common.py:1350:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_modeling_common.py:1455: in _create_and_check_torchscript
    self.fail("Couldn't trace module.")
E   AssertionError: Couldn't trace module.
```

Changed files:
filename: tests/test_modeling_common.py
patch: @@ -21,6 +21,7 @@
 import random
 import re
 import tempfile
+import unittest
 import warnings
 from collections import defaultdict
 from contextlib import contextmanager
@@ -1344,17 +1345,20 @@ def test_attention_outputs(self):
                     [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
                 )
 
+    @unittest.skip("many failing tests after #39120. Will fix when the community ask for it.")
     @slow
     def test_torchscript_simple(self):
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
         self._create_and_check_torchscript(config, inputs_dict)
 
+    @unittest.skip("many failing tests after #39120. Will fix when the community ask for it.")
     @slow
     def test_torchscript_output_attentions(self):
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
         config.output_attentions = True
         self._create_and_check_torchscript(config, inputs_dict)
 
+    @unittest.skip("many failing tests after #39120. Will fix when the community ask for it.")
     @slow
     def test_torchscript_output_hidden_state(self):
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()