Repository: pytorch
Issue: libtorch make failed 
Labels: module: build,triaged

### 🐛 Describe the bug

when I use libtorch with cmakelists:
```
cmake_minimum_required(VERSION 3.15)

project(torch_test)
set(APP_NAME torch_test)
set(CMAKE_PREFIX_PATH "/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/")

IF(NOT CMAKE_BUILD_TYPE)
  SET(CMAKE_BUILD_TYPE Release)
ENDIF() 

set(OpenCV_DIR "/usr/local/share/OpenCV")
find_package(OpenCV REQUIRED)

include_directories(
  ${OpenCV_INCLUDE_DIRS}
)


set(CUDA_DIR "/usr/local/cuda-11.1")
find_package(CUDA REQUIRED)


set(Torch_DIR "/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/share/cmake/Torch")
find_package(Torch REQUIRED)

include_directories(
  ${Torch_INCLUDE_DIRS}
)

file(GLOB SOURCE_FILES torch.cpp)
ADD_EXECUTABLE(${APP_NAME} ${SOURCE_FILES})

target_link_libraries(${APP_NAME} ${TORCH_LIBRARIES})
target_link_libraries(
    ${APP_NAME}
    ${OpenCV_LIBS}
    # ${Torch_LIBRARIES}
    ${catkin_LIBRARIES}
)

set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 14)
```

my CPP fiel as this ,it works.

```
#include <torch/script.h>
#include <torch/torch.h>
#include <iostream>
using namespace torch;
using namespace std;


int main()
{
    torch::Tensor output = torch::randn({ 3,2 });
    std::cout <<output<<endl;
    std::cout <<torch::cuda::is_available()<<endl;
    return 0;
}
```

however ,when I use cmakelists as this :
```
cmake_minimum_required(VERSION 3.15)

project(thrust)
set(APP_NAME thrust)


IF(NOT CMAKE_BUILD_TYPE)
  SET(CMAKE_BUILD_TYPE Release)
ENDIF() 

set(CMAKE_PREFIX_PATH "/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/")
set(Torch_DIR "/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/share/cmake/Torch")
find_package(Torch REQUIRED)
include_directories(${Torch_INCLUDE_DIRS})


set(OpenCV_DIR /usr/local/share/OpenCV)
find_package(OpenCV REQUIRED)
include_directories(
  ${OpenCV_INCLUDE_DIRS}
)

find_package(CUDA REQUIRED)
include_directories(${CUDA_INCLUDE_DIRS})
link_directories(${CUDA_LIBRARY_DIRS})
set(CUDA_NVCC_PLAGS ${CUDA_NVCC_PLAGS};-std=c++14;-g;-G;-gencode;arch=compute_30;code=sm_30)



file(GLOB SOURCE_FILES thrusts.cu)
cuda_add_executable(${PROJECT_NAME} ${SOURCE_FILES})

# target_link_libraries(${APP_NAME} ${TORCH_LIBRARIES})
target_link_libraries(
    ${APP_NAME}
    ${TORCH_LIBRARIES}
    ${OpenCV_LIBS}
    ${CUDA_LIBRARIES} 
)

set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 14)

 ```

it got error :

```
-- Caffe2: CUDA detected: 11.1
-- Caffe2: CUDA nvcc is: /usr/local/cuda-11.1/bin/nvcc
-- Caffe2: CUDA toolkit directory: /usr/local/cuda-11.1
-- Caffe2: Header version is: 11.1
-- Found cuDNN: v8.1.1  (include: /usr/local/cuda-11.1/include, library: /usr/local/cuda-11.1/lib64/libcudnn.so)
-- /usr/local/cuda-11.1/lib64/libnvrtc.so shorthash is 1f6b333a
-- Autodetected CUDA architecture(s):  6.1
-- Added CUDA NVCC flags for: -gencode;arch=compute_61,code=sm_61
-- Configuring done
CMake Warning at /home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/share/cmake/Caffe2/Modules_CUDA_fix/upstream/FindCUDA.cmake:1915 (add_executable):
  Cannot generate a safe runtime search path for target thrust because files
  in some directories may conflict with libraries in implicit directories:

    runtime library [libnvToolsExt.so.1] in /usr/lib/x86_64-linux-gnu may be hidden by files in:
      /usr/local/cuda-11.1/lib64

  Some of these libraries may not be found correctly.
Call Stack (most recent call first):
  CMakeLists.txt:31 (cuda_add_executable)


-- Generating done
-- Build files have been written to: /home/hy/project/scripts/cpp_test/build
[ 50%] Building NVCC (Device) object CMakeFiles/thrust.dir/thrust_generated_thrusts.cu.o
/usr/local/opencv/include/opencv4/opencv2/stitching/detail/warpers.hpp(235): warning: overloaded virtual function "cv::detail::PlaneWarper::buildMaps" is only partially overridden in class "cv::detail::AffineWarper"

/usr/local/opencv/include/opencv4/opencv2/stitching/detail/warpers.hpp(235): warning: overloaded virtual function "cv::detail::PlaneWarper::warp" is only partially overridden in class "cv::detail::AffineWarper"

/usr/local/opencv/include/opencv4/opencv2/stitching/detail/blenders.hpp(100): warning: overloaded virtual function "cv::detail::Blender::prepare" is only partially overridden in class "cv::detail::FeatherBlender"

/usr/local/opencv/include/opencv4/opencv2/stitching/detail/blenders.hpp(127): warning: overloaded virtual function "cv::detail::Blender::prepare" is only partially overridden in class "cv::detail::MultiBandBlender"

/usr/local/opencv/include/opencv4/opencv2/stitching/detail/warpers.hpp(235): warning: overloaded virtual function "cv::detail::PlaneWarper::buildMaps" is only partially overridden in class "cv::detail::AffineWarper"

/usr/local/opencv/include/opencv4/opencv2/stitching/detail/warpers.hpp(235): warning: overloaded virtual function "cv::detail::PlaneWarper::warp" is only partially overridden in class "cv::detail::AffineWarper"

/usr/local/opencv/include/opencv4/opencv2/stitching/detail/blenders.hpp(100): warning: overloaded virtual function "cv::detail::Blender::prepare" is only partially overridden in class "cv::detail::FeatherBlender"

/usr/local/opencv/include/opencv4/opencv2/stitching/detail/blenders.hpp(127): warning: overloaded virtual function "cv::detail::Blender::prepare" is only partially overridden in class "cv::detail::MultiBandBlender"

Scanning dependencies of target thrust
[100%] Linking CXX executable thrust
CMakeFiles/thrust.dir/thrust_generated_thrusts.cu.o：在函数‘c10::IValue::toDouble() const’中：
/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/include/ATen/core/ivalue.h:456：对‘c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)’未定义的引用
CMakeFiles/thrust.dir/thrust_generated_thrusts.cu.o：在函数‘c10::IValue::toInt() const’中：
/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/include/ATen/core/ivalue.h:503：对‘c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)’未定义的引用
CMakeFiles/thrust.dir/thrust_generated_thrusts.cu.o：在函数‘c10::IValue::toComplexDouble() const’中：
/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/include/ATen/core/ivalue_inl.h:137：对‘c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)’未定义的引用
CMakeFiles/thrust.dir/thrust_generated_thrusts.cu.o：在函数‘c10::IValue::toTensor() const &’中：
/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/include/ATen/core/ivalue_inl.h:161：对‘c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)’未定义的引用
CMakeFiles/thrust.dir/thrust_generated_thrusts.cu.o：在函数‘c10::intrusive_ptr<c10::TensorImpl, c10::UndefinedTensorImpl>::retain_()’中：
/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/include/c10/util/intrusive_ptr.h:231：对‘c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)’未定义的引用
CMakeFiles/thrust.dir/thrust_generated_thrusts.cu.o:/home/hy/Downloads/libtorch-shared-with-deps-1.8.2+cu111/libtorch/include/c10/util/intrusive_ptr.h:231: 跟着更多未定义的参考到 c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)
collect2: error: ld returned 1 exit status
CMakeFiles/thrust.dir/build.make:162: recipe for target 'thrust' failed
make[2]: *** [thrust] Error 1
CMakeFiles/Makefile2:75: recipe for target 'CMakeFiles/thrust.dir/all' failed
make[1]: *** [CMakeFiles/thrust.dir/all] Error 2
Makefile:83: recipe for target 'all' failed
make: *** [all] Error 2
```


after I  remove torch ,my program works. so I think the problem is in there.but the first example show I can use torch correctly. I dont kown how to solve it. please help me .thank you !

### Versions

Collecting environment information...
PyTorch version: 1.10.1+cu111
Is debug build: False
CUDA used to build PyTorch: 11.1
ROCM used to build PyTorch: N/A

OS: Ubuntu 18.04.5 LTS (x86_64)
GCC version: (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
Clang version: Could not collect
CMake version: version 3.15.3
Libc version: glibc-2.27

Python version: 3.9.7 (default, Sep 16 2021, 13:09:58)  [GCC 7.5.0] (64-bit runtime)
Python platform: Linux-5.4.0-126-generic-x86_64-with-glibc2.27
Is CUDA available: True
CUDA runtime version: 11.1.74
GPU models and configuration: GPU 0: GeForce GTX 1050 Ti
Nvidia driver version: 455.23.05

libtorch: libtorch-shared-with-deps-1.8.2+cu111

cc @malfet @seemethere