kind_.is_prim() INTERNAL ASSERT FAILED at "../torch/csrc/jit/ir/ir.cpp":1098
Labels: oncall: jit, module: cpp

### 🐛 Describe the bug

## I am inferencing using for loop to inference multiple images. However, I got errors when second index is looping.


- what():  kind_.is_prim() INTERNAL ASSERT FAILED at "../torch/csrc/jit/ir/ir.cpp":1098, please report a bug to PyTorch. Only prim ops are allowed to not have a registered operator but aten::mul doesn't have one either. We don't know if this op has side effects.
- Exception raised from hasSideEffects at ../torch/csrc/jit/ir/ir.cpp:1098 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x69 (0x7f50519f3b29 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libc10.so)
frame #1: torch::jit::Node::hasSideEffects() const + 0x319 (0x7f4fe085f429 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #2: <unknown function> + 0x35f6b18 (0x7f4fe08d8b18 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #3: <unknown function> + 0x35f8761 (0x7f4fe08da761 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #4: <unknown function> + 0x35f97a5 (0x7f4fe08db7a5 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #5: torch::jit::EliminateDeadCode(torch::jit::Block*, std::function<void (std::unordered_set<torch::jit::Value const*, std::hash<torch::jit::Value const*>, std::equal_to<torch::jit::Value const*>, std::allocator<torch::jit::Value const*> > const&)>, torch::jit::DCESideEffectPolicy) + 0x213 (0x7f4fe08d2553 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #6: torch::jit::differentiate(std::shared_ptr<torch::jit::Graph>&) + 0xa53 (0x7f4fe0a2af73 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #7: <unknown function> + 0x377e667 (0x7f4fe0a60667 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #8: <unknown function> + 0x377f5e9 (0x7f4fe0a615e9 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #9: <unknown function> + 0x378014e (0x7f4fe0a6214e in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #10: <unknown function> + 0x3752a24 (0x7f4fe0a34a24 in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #11: torch::jit::GraphFunction::operator()(std::vector<c10::IValue, std::allocator<c10::IValue> >, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, c10::IValue, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, c10::IValue> > > const&) + 0x3a (0x7f4fe074f51a in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #12: torch::jit::Method::operator()(std::vector<c10::IValue, std::allocator<c10::IValue> >, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, c10::IValue, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, c10::IValue> > > const&) + 0x16a (0x7f4fe075e9ca in /workspace/ffmpeg/ffmpeg_sources/libtorch-1.8.0/lib/libtorch_cpu.so)
frame #13: torch::jit::Module::forward(std::vector<c10::IValue, std::allocator<c10::IValue> >) + 0x103 (0x563e74dfe037 in ./libtorch-build)
frame #14: FaceGAN::process(cv::Mat) + 0x554 (0x563e74e137f6 in ./libtorch-build)
frame #15: FaceEnhancement::process(cv::Mat) + 0x63c (0x563e74e1ac80 in ./libtorch-build)
frame #16: main + 0x44c (0x563e74defac5 in ./libtorch-build)
frame #17: __libc_start_main + 0xf3 (0x7f4f60a3d0b3 in /usr/lib/x86_64-linux-gnu/libc.so.6)
frame #18: _start + 0x2e (0x563e74def5be in ./libtorch-build)


```cpp
at::Tensor FaceGAN::process(cv::Mat _img) {
  cv::resize(_img, _img, cv::Size(insize, insize));

  at::Tensor input =
      torch::from_blob(_img.data, {_img.rows, _img.cols, 3}, torch::kByte);
  input = input.permute({2, 0, 1}).div(255.0);
  input = (input - 0.5) / 0.5;
  input = input.unsqueeze(0);
  input = input.to(torch::kFloat32).to(torch::Device(torch::kCUDA, device));
  std::vector<torch::jit::IValue> inputs;
  inputs.push_back(input);
  at::Tensor output = module.forward(inputs).toTensor();
  output = output * 0.5 + 0.5;
  output = output.squeeze().detach();
  output = output.permute({1, 2, 0});
  output = output.mul(255.0);
  output = output.clamp(0, 255).to(torch::kU8).to(torch::kCPU);
  output = output.contiguous();
  return output;
}

void main(){
  at::Tensor facebs, landms;
  std::tie(facebs, landms) = facedetector.process(_img);

  for (int i = 0; i < facebs.sizes()[0]; i++) {
    if (facebs[i][4].item().toFloat() < threshold) continue;
    at::Tensor facial5points = landms[i].reshape({2, 5});
    cv::Mat tfm_inv;
    cv::Mat of;

    std::tie(of, tfm_inv) =
        this->warp_and_crop_face(_img, facial5points, reference_5pts, img_size);
    at::Tensor tensor_ef = facegan.process(of);
    cv::Mat resultImg(tensor_ef.size(0), tensor_ef.size(1), CV_8UC3);
    std::memcpy((void*)resultImg.data, tensor_ef.data_ptr(),
                sizeof(torch::kU8) * tensor_ef.numel());
    cvtColor(resultImg, resultImg, cv::COLOR_RGB2BGR);

    std::string save_path = "temp" + std::to_string(i) + ".png";
    cv::imwrite(save_path, resultImg);
}

```

### Versions

PyTorch version: 1.8.2+cu111
Is debug build: False
CUDA used to build PyTorch: 11.1
ROCM used to build PyTorch: N/A

OS: Ubuntu 20.04.3 LTS (x86_64)
GCC version: (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0
Clang version: Could not collect
CMake version: version 3.14.4
Libc version: glibc-2.31

Python version: 3.8.10 (default, Sep 28 2021, 16:10:42)  [GCC 9.3.0] (64-bit runtime)
Python platform: Linux-4.18.0-348.20.1.el8_5.x86_64-x86_64-with-glibc2.29
Is CUDA available: True
CUDA runtime version: 11.5.50
CUDA_MODULE_LOADING set to: 
GPU models and configuration: 
GPU 0: NVIDIA A100-SXM4-40GB
GPU 1: NVIDIA A100-SXM4-40GB
GPU 2: NVIDIA A100-SXM4-40GB
GPU 3: NVIDIA A100-SXM4-40GB
GPU 4: NVIDIA A100-SXM4-40GB
GPU 5: NVIDIA A100-SXM4-40GB
GPU 6: NVIDIA A100-SXM4-40GB
GPU 7: NVIDIA A100-SXM4-40GB

Nvidia driver version: 470.103.01
cuDNN version: Probably one of the following:
/usr/lib/x86_64-linux-gnu/libcudnn.so.8.3.1
/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.3.1
/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.3.1
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.3.1
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.3.1
/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.3.1
/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.3.1
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

Versions of relevant libraries:
[pip3] numpy==1.21.4
[pip3] torch==1.8.2+cu111
[pip3] torchaudio==0.8.2
[pip3] torchvision==0.9.2+cu111
[conda] Could not collect

cc @EikanWang @jgong5 @wenzhe-nrv @sanchitintel @jbschlosser