Repository: pytorch
Issue: XLA not being tested in TestAutogradDeviceType
Labels: module: autograd,module: tests,triaged,module: xla

Moved a test from TestAutogradDeviceType (test_autograd.py) to TestNNDeviceType (test_nn.py) and discovered this failure in CI https://app.circleci.com/pipelines/github/pytorch/pytorch/401973/workflows/8fda9f8d-e75d-41f5-8462-63d2bbdc8498/jobs/16615699

From the XLA code it looks like we don't support the inplace case when negative_slope is zero or negative, so it is possible this particular failure is expected:
```
at::Tensor XLANativeFunctions::leaky_relu_backward(
    const at::Tensor& grad_output, const at::Tensor& self,
    const at::Scalar& negative_slope, bool self_is_result) {
  XLA_FN_COUNTER("xla::");
  XLA_CHECK(!self_is_result || negative_slope.to<double>() > 0.0);
  return bridge::AtenFromXlaTensor(XLATensor::leaky_relu_backward(
      bridge::GetXlaTensor(grad_output), bridge::GetXlaTensor(self),
      negative_slope.to<double>()));
}
```
The question is whether it is expected that XLA wasn't being tested in TestAutogradDeviceType, but it is in TestNNDeviceType.
```
======================================================================
Oct 28 22:06:26 ERROR [0.012s]: test_leaky_relu_inplace_with_zero_slope_xla (__main__.TestNNDeviceTypeXLA)
Oct 28 22:06:26 ----------------------------------------------------------------------
Oct 28 22:06:26 Traceback (most recent call last):
Oct 28 22:06:26   File "/opt/conda/lib/python3.6/site-packages/torch/testing/_internal/common_device_type.py", line 376, in instantiated_test
Oct 28 22:06:26     raise rte
Oct 28 22:06:26   File "/opt/conda/lib/python3.6/site-packages/torch/testing/_internal/common_device_type.py", line 371, in instantiated_test
Oct 28 22:06:26     result = test(self, **param_kwargs)
Oct 28 22:06:26   File "/var/lib/jenkins/workspace/xla/test/../../test/test_nn.py", line 17986, in test_leaky_relu_inplace_with_zero_slope
Oct 28 22:06:26     b.backward(torch.ones(3, device=device))
Oct 28 22:06:26   File "/opt/conda/lib/python3.6/site-packages/torch/_tensor.py", line 352, in backward
Oct 28 22:06:26     torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
Oct 28 22:06:26   File "/opt/conda/lib/python3.6/site-packages/torch/autograd/__init__.py", line 175, in backward
Oct 28 22:06:26     allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass
Oct 28 22:06:26 RuntimeError: /var/lib/jenkins/workspace/xla/torch_xla/csrc/aten_xla_type.cpp:1744 : Check failed: !self_is_result || negative_slope.to<double>() > 0.0 
Oct 28 22:06:26 *** Begin stack trace ***
Oct 28 22:06:26 	tensorflow::CurrentStackTrace[abi:cxx11]()
Oct 28 22:06:26 	torch_xla::XLANativeFunctions::leaky_relu_backward(at::Tensor const&, at::Tensor const&, c10::Scalar const&, bool)
Oct 28 22:06:26 	
Oct 28 22:06:26 	
Oct 28 22:06:26 	at::_ops::leaky_relu_backward::redispatch(c10::DispatchKeySet, at::Tensor const&, at::Tensor const&, c10::Scalar const&, bool)
Oct 28 22:06:26 	
Oct 28 22:06:26 	at::_ops::leaky_relu_backward::call(at::Tensor const&, at::Tensor const&, c10::Scalar const&, bool)
Oct 28 22:06:26 	torch::autograd::generated::LeakyReluBackward1::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&)
Oct 28 22:06:26 	
Oct 28 22:06:26 	torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&, std::shared_ptr<torch::autograd::ReadyQueue> const&)
Oct 28 22:06:26 	torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&)
Oct 28 22:06:26 	torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool)
Oct 28 22:06:26 	torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool)
Oct 28 22:06:26 	
Oct 28 22:06:26 	
Oct 28 22:06:26 	clone
Oct 28 22:06:26 *** End stack trace ***
Oct 28 22:06:26 
Oct 28 22:06:26 
Oct 28 22:06:26 ======================================================================
Oct 28 22:06:26 FAIL [0.015s]: test_leaky_relu_inplace_with_neg_slope_xla (__main__.TestNNDeviceTypeXLA)
Oct 28 22:06:26 ----------------------------------------------------------------------
Oct 28 22:06:26 RuntimeError: /var/lib/jenkins/workspace/xla/torch_xla/csrc/aten_xla_type.cpp:1744 : Check failed: !self_is_result || negative_slope.to<double>() > 0.0 
Oct 28 22:06:26 *** Begin stack trace ***
Oct 28 22:06:26 	tensorflow::CurrentStackTrace[abi:cxx11]()
Oct 28 22:06:26 	torch_xla::XLANativeFunctions::leaky_relu_backward(at::Tensor const&, at::Tensor const&, c10::Scalar const&, bool)
Oct 28 22:06:26 	
Oct 28 22:06:26 	
Oct 28 22:06:26 	at::_ops::leaky_relu_backward::redispatch(c10::DispatchKeySet, at::Tensor const&, at::Tensor const&, c10::Scalar const&, bool)
Oct 28 22:06:26 	
Oct 28 22:06:26 	at::_ops::leaky_relu_backward::call(at::Tensor const&, at::Tensor const&, c10::Scalar const&, bool)
Oct 28 22:06:26 	torch::autograd::generated::LeakyReluBackward1::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&)
Oct 28 22:06:26 	
Oct 28 22:06:26 	torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&, std::shared_ptr<torch::autograd::ReadyQueue> const&)
Oct 28 22:06:26 	torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&)
Oct 28 22:06:26 	torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool)
Oct 28 22:06:26 	torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool)
Oct 28 22:06:26 	
Oct 28 22:06:26 	
Oct 28 22:06:26 	clone
Oct 28 22:06:26 *** End stack trace ***
Oct 28 22:06:26 
Oct 28 22:06:26 
Oct 28 22:06:26 During handling of the above exception, another exception occurred:
Oct 28 22:06:26 
Oct 28 22:06:26 Traceback (most recent call last):
Oct 28 22:06:26   File "/opt/conda/lib/python3.6/site-packages/torch/testing/_internal/common_device_type.py", line 371, in instantiated_test
Oct 28 22:06:26     result = test(self, **param_kwargs)
Oct 28 22:06:26   File "/var/lib/jenkins/workspace/xla/test/../../test/test_nn.py", line 17976, in test_leaky_relu_inplace_with_neg_slope
Oct 28 22:06:26     b.backward(torch.ones(2, device=device))
Oct 28 22:06:26 AssertionError: "call out-of-place version" does not match "/var/lib/jenkins/workspace/xla/torch_xla/csrc/aten_xla_type.cpp:1744 : Check failed: !self_is_result || negative_slope.to<double>() > 0.0 
Oct 28 22:06:26 *** Begin stack trace ***
Oct 28 22:06:26 	tensorflow::CurrentStackTrace[abi:cxx11]()
Oct 28 22:06:26 	torch_xla::XLANativeFunctions::leaky_relu_backward(at::Tensor const&, at::Tensor const&, c10::Scalar const&, bool)
Oct 28 22:06:26 	
Oct 28 22:06:26 	
Oct 28 22:06:26 	at::_ops::leaky_relu_backward::redispatch(c10::DispatchKeySet, at::Tensor const&, at::Tensor const&, c10::Scalar const&, bool)
Oct 28 22:06:26 	
Oct 28 22:06:26 	at::_ops::leaky_relu_backward::call(at::Tensor const&, at::Tensor const&, c10::Scalar const&, bool)
Oct 28 22:06:26 	torch::autograd::generated::LeakyReluBackward1::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&)
Oct 28 22:06:26 	
Oct 28 22:06:26 	torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&, std::shared_ptr<torch::autograd::ReadyQueue> const&)
Oct 28 22:06:26 	torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&)
Oct 28 22:06:26 	torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool)
Oct 28 22:06:26 	torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool)
Oct 28 22:06:26 	
Oct 28 22:06:26 	
Oct 28 22:06:26 	clone
Oct 28 22:06:26 *** End stack trace ***
Oct 28 22:06:26 "
```

cc @ezyang @albanD @zou3519 @gqchen @pearu @nikitaved @soulitzer @Lezcano @Varal7 @mruberry @bdhirsh