Repository: pytorch/pytorch

Issue #93599: Inductor - resnet18 -  large batch size - CUDA error: an illegal memory access was encountered 
Created: 2022-11-01T19:59:40Z
🐛 Describe the bug
Repro:
import torch
import torch._dynamo
import torch._inductor
from torch._inductor import config
import logging
from torchvision import models

resnet18 = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)

batch_size = 4096
device = "cuda"

resnet18 = resnet18.eval().to(device)
opt_resnet18 = torch._dynamo.optimize("inductor")(resnet18)

input = torch.randn((batch_size, 3, 224, 224)).to(device)
output = opt_resnet18(input)
print(output.shape)

This only happens when batch size is large.
Error logs
Traceback (most recent call last):
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/utils.py", line 458, in preserve_rng_state
    yield
  File "/scratch/ybliang/work/repos/pytorch/torch/_inductor/compile_fx.py", line 202, in run
    compiled_fn = cudagraphify_impl(model, new_inputs, static_input_idxs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_inductor/compile_fx.py", line 257, in cudagraphify_impl
    model(list(static_inputs))
  File "/tmp/torchinductor_ybliang/7q/c7qimro7rryowl6fbgxobggppym6ux4mwk4x5htmdqso66ydxlb3.py", line 691, in call
    buf3 = empty_strided((4096, 64, 56, 56), (200704, 3136, 56, 1), device='cuda', dtype=torch.int64)
RuntimeError: CUDA error: an illegal memory access was encountered
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/scratch/ybliang/work/repos/pytorch/debug/debug5.py", line 35, in <module>
    output = opt_resnet18(input)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/eval_frame.py", line 138, in __call__
    return self.forward(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/eval_frame.py", line 135, in forward
    return optimized_forward(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/eval_frame.py", line 166, in _fn
    return fn(*args, **kwargs)
  File "/scratch/ybliang/work/repos/torchvision/torchvision/models/resnet.py", line 284, in forward
    def forward(self, x: Tensor) -> Tensor:
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/eval_frame.py", line 166, in _fn
    return fn(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/functorch/_src/aot_autograd.py", line 870, in forward
    return compiled_f(
  File "/scratch/ybliang/work/repos/pytorch/functorch/_src/aot_autograd.py", line 861, in new_func
    return compiled_fn(args)
  File "/scratch/ybliang/work/repos/pytorch/functorch/_src/aot_autograd.py", line 230, in g
    return f(*args)
  File "/scratch/ybliang/work/repos/pytorch/functorch/_src/aot_autograd.py", line 489, in compiled_function
    return CompiledFunction.apply(*remove_dupe_args(args))
  File "/scratch/ybliang/work/repos/pytorch/functorch/_src/aot_autograd.py", line 450, in forward
    fw_outs = call_func_with_args(
  File "/scratch/ybliang/work/repos/pytorch/functorch/_src/aot_autograd.py", line 255, in call_func_with_args
    out = normalize_as_list(f(args))
  File "/scratch/ybliang/work/repos/pytorch/torch/_inductor/compile_fx.py", line 185, in run
    return model(new_inputs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_inductor/compile_fx.py", line 202, in run
    compiled_fn = cudagraphify_impl(model, new_inputs, static_input_idxs)
  File "/scratch/ybliang/work/env/lib/python3.9/contextlib.py", line 137, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/utils.py", line 462, in preserve_rng_state
    torch.cuda.set_rng_state(cuda_rng)
  File "/scratch/ybliang/work/repos/pytorch/torch/cuda/random.py", line 64, in set_rng_state
    _lazy_call(cb)
  File "/scratch/ybliang/work/repos/pytorch/torch/cuda/__init__.py", line 176, in _lazy_call
    callable()
  File "/scratch/ybliang/work/repos/pytorch/torch/cuda/random.py", line 62, in cb
    default_generator.set_state(new_state_copy)
RuntimeError: CUDA error: an illegal memory access was encountered
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.

Minified repro
No response
cc @ezyang @soumith @msaroufim @wconstab @ngimel @bdhirsh

Duplicate issue #93590: test_conv_large_cuda: RuntimeError: CUDA error: an illegal memory access was encountered
Created: 2022-10-28T00:05:20Z
Repro:
PYTORCH_TEST_WITH_INDUCTOR=1 python test/nn/test_convolution.py  -k test_conv_large_cuda

Error:
TEST SUITE EARLY TERMINATION due to torch.cuda.synchronize() failure
ETEST SUITE EARLY TERMINATION due to torch.cuda.synchronize() failure

======================================================================
ERROR: test_conv_large_cuda (__main__.TestNNDeviceTypeCUDA)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/fsx/users/binbao/pytorch/torch/_dynamo/utils.py", line 442, in preserve_rng_state
    yield
  File "/fsx/users/binbao/pytorch/torch/_inductor/compile_fx.py", line 203, in run
    compiled_fn = cudagraphify_impl(model, new_inputs, static_input_idxs)
  File "/fsx/users/binbao/pytorch/torch/_inductor/compile_fx.py", line 258, in cudagraphify_impl
    model(list(static_inputs))
  File "/tmp/torchinductor_binbao/5x/c5xaannepkzj65rsuvrsfwf6kviuwaz4eic3zumxo7nerrkxkvgh.py", line 55, in call
    buf1 = aten.convolution(buf2, primals_1, None, (8, 8), (0, 0), (1, 1), False, (0, 0), 1)
  File "/fsx/users/binbao/pytorch/torch/_ops.py", line 446, in __call__
    return self._op(*args, **kwargs or {})
RuntimeError: cuDNN error: CUDNN_STATUS_INTERNAL_ERROR

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/fsx/users/binbao/pytorch/torch/testing/_internal/common_utils.py", line 2001, in wrapper
    method(*args, **kwargs)
  File "/fsx/users/binbao/pytorch/torch/testing/_internal/common_utils.py", line 2001, in wrapper
    method(*args, **kwargs)
  File "/fsx/users/binbao/pytorch/torch/testing/_internal/common_device_type.py", line 394, in instantiated_test
    raise rte
  File "/fsx/users/binbao/pytorch/torch/testing/_internal/common_device_type.py", line 381, in instantiated_test
    result = test(self, **param_kwargs)
  File "/fsx/users/binbao/pytorch/torch/testing/_internal/common_device_type.py", line 991, in only_fn
    return fn(slf, *args, **kwargs)
  File "/fsx/users/binbao/pytorch/torch/testing/_internal/common_device_type.py", line 869, in dep_fn
    return fn(slf, *args, **kwargs)
  File "/fsx/users/binbao/pytorch/torch/testing/_internal/common_device_type.py", line 952, in dep_fn
    return fn(self, *args, **kwargs)
  File "/fsx/users/binbao/pytorch/test/test_nn.py", line 15390, in test_conv_large
    conv = nn.Conv2d(2, 2, 8, 8, bias=False).to(device).to(dtype)
  File "/fsx/users/binbao/pytorch/test/test_nn.py", line 15390, in <graph break in test_conv_large>
    conv = nn.Conv2d(2, 2, 8, 8, bias=False).to(device).to(dtype)
  File "/fsx/users/binbao/pytorch/test/test_nn.py", line 15390, in <graph break in test_conv_large>
    conv = nn.Conv2d(2, 2, 8, 8, bias=False).to(device).to(dtype)
  File "/fsx/users/binbao/pytorch/test/test_nn.py", line 15390, in <graph break in test_conv_large>
    conv = nn.Conv2d(2, 2, 8, 8, bias=False).to(device).to(dtype)
  File "/fsx/users/binbao/pytorch/torch/_dynamo/eval_frame.py", line 157, in _fn
    return fn(*args, **kwargs)
  File "/fsx/users/binbao/pytorch/functorch/_src/aot_autograd.py", line 870, in forward
    return compiled_f(
  File "/fsx/users/binbao/pytorch/functorch/_src/aot_autograd.py", line 861, in new_func
    return compiled_fn(args)
  File "/fsx/users/binbao/pytorch/functorch/_src/aot_autograd.py", line 230, in g
    return f(*args)
  File "/fsx/users/binbao/pytorch/functorch/_src/aot_autograd.py", line 489, in compiled_function
    return CompiledFunction.apply(*remove_dupe_args(args))
  File "/fsx/users/binbao/torchdynamo-tip/torchdynamo/eval_frame.py", line 163, in _fn
    return fn(*args, **kwargs)
  File "/fsx/users/binbao/pytorch/functorch/_src/aot_autograd.py", line 450, in forward
    fw_outs = call_func_with_args(
  File "/fsx/users/binbao/pytorch/functorch/_src/aot_autograd.py", line 255, in call_func_with_args
    out = normalize_as_list(f(args))
  File "/fsx/users/binbao/pytorch/torch/_inductor/compile_fx.py", line 186, in run
    return model(new_inputs)
  File "/fsx/users/binbao/pytorch/torch/_inductor/compile_fx.py", line 203, in run
    compiled_fn = cudagraphify_impl(model, new_inputs, static_input_idxs)
  File "/fsx/users/binbao/conda/envs/release/lib/python3.9/contextlib.py", line 137, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/fsx/users/binbao/pytorch/torch/_dynamo/utils.py", line 446, in preserve_rng_state
    torch.cuda.set_rng_state(cuda_rng)
  File "/fsx/users/binbao/pytorch/torch/cuda/random.py", line 64, in set_rng_state
    _lazy_call(cb)
  File "/fsx/users/binbao/pytorch/torch/cuda/__init__.py", line 176, in _lazy_call
    callable()
  File "/fsx/users/binbao/pytorch/torch/cuda/random.py", line 62, in cb
    default_generator.set_state(new_state_copy)
RuntimeError: CUDA error: an illegal memory access was encountered
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.

Closing comment (2022-11-02T19:10:42Z): I think this is a dupe of #93599. Root cause: triton-lang/triton#832