Repository: pytorch
Issue: cpp_wrapper and triton.debug_sync_kernel errors out
Labels: triaged,oncall: pt2,module: inductor

### 🐛 Describe the bug

Similar to https://github.com/pytorch/pytorch/issues/140219 these options also fail.
Code:
```
import torch
batch_size = 32
seq_length = 50
hidden_size = 768

def test_fn():
    inp = torch.randn(batch_size, seq_length, hidden_size, device="cuda")
    weight = torch.randn(hidden_size, hidden_size, device="cuda")
    matmul_output = inp @ weight
    final_output = torch.nn.LayerNorm(hidden_size, device="cuda")(matmul_output)
    return final_output
comp = torch.compile(options={"cpp_wrapper": True, "triton.debug_sync_kernel": True, "force_disable_caches": True})(test_fn)
print(comp())

```

### Error logs
```
/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py:222: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
  warnings.warn(
W1109 17:50:43.704000 2516607 torch/_inductor/utils.py:847] [0/0_1] on error, temporary cache dir kept at /tmp/torchinductor_gabeferns/tmp2x69j_al
Traceback (most recent call last):
  File "/home/gabeferns/org/debug/config-fuzzer-1/repro2.py", line 13, in <module>
    print(comp())
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/eval_frame.py", line 556, in _fn
    return fn(*args, **kwargs)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/convert_frame.py", line 1447, in __call__
    return self._torchdynamo_orig_callable(
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/convert_frame.py", line 1232, in __call__
    result = self._inner_convert(
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/convert_frame.py", line 550, in __call__
    return _compile(
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/convert_frame.py", line 979, in _compile
    guarded_code = compile_inner(code, one_graph, hooks, transform)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/convert_frame.py", line 709, in compile_inner
    return _compile_inner(code, one_graph, hooks, transform)
  File "/home/gabeferns/pt-envs/applications/torch/_utils_internal.py", line 95, in wrapper_function
    return function(*args, **kwargs)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/convert_frame.py", line 744, in _compile_inner
    out_code = transform_code_object(code, transform)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/bytecode_transformation.py", line 1348, in transform_code_object
    transformations(instructions, code_options)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/convert_frame.py", line 234, in _fn
    return fn(*args, **kwargs)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/convert_frame.py", line 663, in transform
    tracer.run()
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/symbolic_convert.py", line 2914, in run
    super().run()
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/symbolic_convert.py", line 1120, in run
    while self.step():
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/symbolic_convert.py", line 1032, in step
    self.dispatch_table[inst.opcode](self, inst)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/symbolic_convert.py", line 638, in wrapper
    return handle_graph_break(self, inst, speculation.reason)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/symbolic_convert.py", line 679, in handle_graph_break
    self.output.compile_subgraph(self, reason=reason)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/output_graph.py", line 1110, in compile_subgraph
    self.compile_and_call_fx_graph(tx, pass2.graph_output_vars(), root)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/output_graph.py", line 1346, in compile_and_call_fx_graph
    compiled_fn = self.call_user_compiler(gm)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/output_graph.py", line 1395, in call_user_compiler
    return self._call_user_compiler(gm)
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/output_graph.py", line 1444, in _call_user_compiler
    raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/output_graph.py", line 1425, in _call_user_compiler
    compiled_fn = compiler_fn(gm, self.example_inputs())
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/repro/after_dynamo.py", line 130, in __call__
    compiled_gm = compiler_fn(gm, example_inputs)
  File "/home/gabeferns/pt-envs/applications/torch/__init__.py", line 2300, in __call__
    return compile_fx(model_, inputs_, config_patches=self.config)
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py", line 1427, in compile_fx
    return compile_fx(
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py", line 1469, in compile_fx
    return compile_fx(
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py", line 1707, in compile_fx
    return aot_autograd(
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/backends/common.py", line 72, in __call__
    cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
  File "/home/gabeferns/pt-envs/applications/torch/_functorch/aot_autograd.py", line 1103, in aot_module_simplified
    compiled_fn = dispatch_and_compile()
  File "/home/gabeferns/pt-envs/applications/torch/_functorch/aot_autograd.py", line 1079, in dispatch_and_compile
    compiled_fn, _ = create_aot_dispatcher_function(
  File "/home/gabeferns/pt-envs/applications/torch/_functorch/aot_autograd.py", line 527, in create_aot_dispatcher_function
    return _create_aot_dispatcher_function(
  File "/home/gabeferns/pt-envs/applications/torch/_functorch/aot_autograd.py", line 778, in _create_aot_dispatcher_function
    compiled_fn, fw_metadata = compiler_fn(
  File "/home/gabeferns/pt-envs/applications/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line 197, in aot_dispatch_base
    compiled_fw = compiler(fw_module, updated_flat_args)
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py", line 1524, in fw_compiler_base
    return _fw_compiler_base(model, example_inputs, is_inference)
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py", line 1593, in _fw_compiler_base
    return inner_compile(
  File "/home/gabeferns/.conda/envs/applications/lib/python3.10/contextlib.py", line 79, in inner
    return func(*args, **kwds)
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py", line 587, in compile_fx_inner
    return wrap_compiler_debug(_compile_fx_inner, compiler_name="inductor")(
  File "/home/gabeferns/pt-envs/applications/torch/_dynamo/repro/after_aot.py", line 102, in debug_wrapper
    inner_compiled_fn = compiler_fn(gm, example_inputs)
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py", line 754, in _compile_fx_inner
    compiled_graph = codegen_and_compile(
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py", line 651, in codegen_and_compile
    compiled_graph = fx_codegen_and_compile(gm, example_inputs, **fx_kwargs)
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/compile_fx.py", line 962, in fx_codegen_and_compile
    compiled_fn = graph.compile_to_fn()
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/graph.py", line 2034, in compile_to_fn
    return self.compile_to_module().call
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/graph.py", line 1954, in compile_to_module
    return self._compile_to_module()
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/graph.py", line 1988, in _compile_to_module
    mod = PyCodeCache.load_by_key_path(
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/codecache.py", line 3057, in load_by_key_path
    mod = _reload_python_module(key, path)
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/runtime/compile_tasks.py", line 45, in _reload_python_module
    exec(code, mod.__dict__, mod.__dict__)
  File "/tmp/torchinductor_gabeferns/tmp2x69j_al/wr/cwrrzoj2tdotjlwmc6zqlcw5xxv6ghfwuwhz2rdvnwbuijycrfux.py", line 221, in <module>
    inductor_entry = CppWrapperCodeCache.load_pybinding(
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/codecache.py", line 2562, in load_pybinding
    return cls.load_pybinding_async(*args, **kwargs)()
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/codecache.py", line 2554, in future
    result = get_result()
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/codecache.py", line 2345, in load_fn
    result = worker_fn()
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/codecache.py", line 2385, in _worker_compile_cpp
    cpp_builder.build()
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/cpp_builder.py", line 1528, in build
    status = run_compile_cmd(build_cmd, cwd=_build_tmp_dir)
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/cpp_builder.py", line 350, in run_compile_cmd
    return _run_compile_cmd(cmd_line, cwd)
  File "/home/gabeferns/pt-envs/applications/torch/_inductor/cpp_builder.py", line 344, in _run_compile_cmd
    raise exc.CppCompileError(cmd, output) from e
torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
CppCompileError: C++ compile error

Command:
g++ /tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp -D TORCH_INDUCTOR_CPP_WRAPPER -D STANDALONE_TORCH_HEADER -D C10_USING_CUSTOM_GENERATED_MACROS -D CPU_CAPABILITY_AVX512 -D USE_CUDA -shared -fPIC -O3 -DNDEBUG -fno-trapping-math -funsafe-math-optimizations -ffinite-math-only -fno-signed-zeros -fno-math-errno -fexcess-precision=fast -fno-finite-math-only -fno-unsafe-math-optimizations -ffp-contract=off -fno-tree-loop-vectorize -march=native -Wall -std=c++17 -Wno-unused-variable -Wno-unknown-pragmas -fopenmp -I/home/gabeferns/.conda/envs/applications/include/python3.10 -I/home/gabeferns/.conda/envs/applications/include/python3.10 -I/home/gabeferns/pt-envs/applications/torch/include -I/home/gabeferns/pt-envs/applications/torch/include/torch/csrc/api/include -I/home/gabeferns/pt-envs/applications/torch/include/TH -I/home/gabeferns/pt-envs/applications/torch/include/THC -I/home/gabeferns/pt-envs/applications/torch/include -I/home/gabeferns/pt-envs/applications/torch/include/torch/csrc/api/include -I/home/gabeferns/pt-envs/applications/torch/include/TH -I/home/gabeferns/pt-envs/applications/torch/include/THC -I/usr/local/cuda-12.4/include -mavx512f -mavx512dq -mavx512vl -mavx512bw -mfma -D_GLIBCXX_USE_CXX11_ABI=1 -ltorch -ltorch_cpu -ltorch_python -lgomp -lc10_cuda -lcuda -ltorch_cuda -L/home/gabeferns/.conda/envs/applications/lib -L/home/gabeferns/pt-envs/applications/torch/lib -L/home/gabeferns/pt-envs/applications/torch/lib -L/usr/local/cuda-12.4/lib64 -o /tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.so

Output:
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:135:131: warning: integer constant is so large that it is unsigned
  135 |     AOTI_TORCH_ERROR_CODE_CHECK(aoti_torch_cuda_randint_low_out(buf0, -9223372036854775808L, 9223372036854775807L, var_array_0, 1));
      |                                                                                                                                   ^
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp: In function ‘void inductor_entry_impl(AtenTensorOpaque**, AtenTensorOpaque**)’:
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:136:10: error: expected primary-expression before ‘.’ token
  136 |     torch.cuda.synchronize()
      |          ^
In file included from /tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:31:
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:140:61: error: ‘int_array_4’ was not declared in this scope; did you mean ‘int_array_5’?
  140 |     AOTI_TORCH_ERROR_CODE_CHECK(aoti_torch_empty_strided(3, int_array_4, int_array_5, cached_torch_dtype_float32, cached_torch_device_type_cuda,  0, &buf1_handle));
      |                                                             ^~~~~~~~~~~
/home/gabeferns/pt-envs/applications/torch/include/torch/csrc/inductor/aoti_runtime/utils.h:34:8: note: in definition of macro ‘AOTI_TORCH_ERROR_CODE_CHECK’
   34 |   if ((call) != AOTI_TORCH_SUCCESS) {           \
      |        ^~~~
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:159:10: error: expected primary-expression before ‘.’ token
  159 |     torch.cuda.synchronize()
      |          ^
In file included from /tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:31:
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:163:61: error: ‘int_array_6’ was not declared in this scope; did you mean ‘int_array_7’?
  163 |     AOTI_TORCH_ERROR_CODE_CHECK(aoti_torch_empty_strided(2, int_array_6, int_array_7, cached_torch_dtype_float32, cached_torch_device_type_cuda,  0, &buf2_handle));
      |                                                             ^~~~~~~~~~~
/home/gabeferns/pt-envs/applications/torch/include/torch/csrc/inductor/aoti_runtime/utils.h:34:8: note: in definition of macro ‘AOTI_TORCH_ERROR_CODE_CHECK’
   34 |   if ((call) != AOTI_TORCH_SUCCESS) {           \
      |        ^~~~
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:180:10: error: expected primary-expression before ‘.’ token
  180 |     torch.cuda.synchronize()
      |          ^
In file included from /tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:31:
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:183:61: error: ‘int_array_8’ was not declared in this scope; did you mean ‘int_array_7’?
  183 |     AOTI_TORCH_ERROR_CODE_CHECK(aoti_torch_empty_strided(2, int_array_8, int_array_7, cached_torch_dtype_float32, cached_torch_device_type_cuda,  0, &buf3_handle));
      |                                                             ^~~~~~~~~~~
/home/gabeferns/pt-envs/applications/torch/include/torch/csrc/inductor/aoti_runtime/utils.h:34:8: note: in definition of macro ‘AOTI_TORCH_ERROR_CODE_CHECK’
   34 |   if ((call) != AOTI_TORCH_SUCCESS) {           \
      |        ^~~~
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:192:10: error: expected primary-expression before ‘.’ token
  192 |     torch.cuda.synchronize()
      |          ^
/tmp/torchinductor_gabeferns/tmp2x69j_al/kl/ckl476gfctiaop3746j24yrqisr24wgiba7mlpvu7qq6q5jjkzjd.cpp:194:57: error: ‘tmp_tensor_handle_1’ was not declared in this scope; did you mean ‘tmp_tensor_handle_0’?
  194 |     output_handles[0] = wrap_with_raii_handle_if_needed(tmp_tensor_handle_1).release();
      |                                                         ^~~~~~~~~~~~~~~~~~~
      |                                                         tmp_tensor_handle_0


Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information


You can suppress this exception and fall back to eager by setting:
    import torch._dynamo
    torch._dynamo.config.suppress_errors = True

```

### Versions

Collecting environment information...
PyTorch version: 2.6.0a0+git13eb3b3
Is debug build: True
CUDA used to build PyTorch: 12.4
ROCM used to build PyTorch: N/A

OS: CentOS Stream 9 (x86_64)
GCC version: (GCC) 11.5.0 20240719 (Red Hat 11.5.0-2)
Clang version: Could not collect
CMake version: version 3.26.4
Libc version: glibc-2.34

Python version: 3.10.14 (main, May  6 2024, 19:42:50) [GCC 11.2.0] (64-bit runtime)
Python platform: Linux-6.4.3-0_fbk14_hardened_2601_gcd42476b84e9-x86_64-with-glibc2.34
Is CUDA available: True
CUDA runtime version: 12.4.131
CUDA_MODULE_LOADING set to: LAZY
GPU models and configuration: GPU 0: NVIDIA H100
Nvidia driver version: 550.90.07
cuDNN version: Could not collect
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: False

CPU:
Architecture:                       x86_64
CPU op-mode(s):                     32-bit, 64-bit
Address sizes:                      52 bits physical, 57 bits virtual
Byte Order:                         Little Endian
CPU(s):                             46
On-line CPU(s) list:                0-45
Vendor ID:                          AuthenticAMD
Model name:                         AMD EPYC 9654 96-Core Processor
CPU family:                         25
Model:                              17
Thread(s) per core:                 1
Core(s) per socket:                 46
Socket(s):                          1
Stepping:                           1
BogoMIPS:                           4792.78
Flags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm rep_good nopl cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw perfctr_core invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx512_bf16 clzero xsaveerptr wbnoinvd arat npt lbrv nrip_save tsc_scale vmcb_clean pausefilter pfthreshold v_vmsave_vmload vgif vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm flush_l1d arch_capabilities
Virtualization:                     AMD-V
Hypervisor vendor:                  KVM
Virtualization type:                full
L1d cache:                          2.9 MiB (46 instances)
L1i cache:                          2.9 MiB (46 instances)
L2 cache:                           23 MiB (46 instances)
L3 cache:                           736 MiB (46 instances)
NUMA node(s):                       1
NUMA node0 CPU(s):                  0-45
Vulnerability Gather data sampling: Not affected
Vulnerability Itlb multihit:        Not affected
Vulnerability L1tf:                 Not affected
Vulnerability Mds:                  Not affected
Vulnerability Meltdown:             Not affected
Vulnerability Mmio stale data:      Not affected
Vulnerability Retbleed:             Not affected
Vulnerability Spec store bypass:    Vulnerable
Vulnerability Spectre v1:           Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers
Vulnerability Spectre v2:           Vulnerable, IBPB: disabled, STIBP: disabled, PBRSB-eIBRS: Not affected
Vulnerability Srbds:                Not affected
Vulnerability Tsx async abort:      Not affected

Versions of relevant libraries:
[pip3] flake8==6.1.0
[pip3] flake8-bugbear==23.3.23
[pip3] flake8-comprehensions==3.15.0
[pip3] flake8-executable==2.1.3
[pip3] flake8-logging-format==0.9.0
[pip3] flake8-pyi==23.3.1
[pip3] flake8-simplify==0.19.3
[pip3] mypy==1.11.2
[pip3] mypy-extensions==1.0.0
[pip3] numpy==1.26.0
[pip3] nvidia-cublas-cu12==12.4.5.8
[pip3] nvidia-cuda-cupti-cu12==12.4.127
[pip3] nvidia-cuda-nvrtc-cu12==12.4.127
[pip3] nvidia-cuda-runtime-cu12==12.4.127
[pip3] nvidia-cudnn-cu12==9.1.0.70
[pip3] nvidia-cufft-cu12==11.2.1.3
[pip3] nvidia-curand-cu12==10.3.5.147
[pip3] nvidia-cusolver-cu12==11.6.1.9
[pip3] nvidia-cusparse-cu12==12.3.1.170
[pip3] nvidia-nccl-cu12==2.21.5
[pip3] nvidia-nvjitlink-cu12==12.4.127
[pip3] nvidia-nvtx-cu12==12.4.127
[pip3] optree==0.13.0
[pip3] pytorch-triton==3.0.0+dedb7bdf33
[pip3] torch==2.6.0a0+git13eb3b3
[pip3] triton==3.1.0
[conda] numpy                     1.26.0                   pypi_0    pypi
[conda] nvidia-cublas-cu12        12.4.5.8                 pypi_0    pypi
[conda] nvidia-cuda-cupti-cu12    12.4.127                 pypi_0    pypi
[conda] nvidia-cuda-nvrtc-cu12    12.4.127                 pypi_0    pypi
[conda] nvidia-cuda-runtime-cu12  12.4.127                 pypi_0    pypi
[conda] nvidia-cudnn-cu12         9.1.0.70                 pypi_0    pypi
[conda] nvidia-cufft-cu12         11.2.1.3                 pypi_0    pypi
[conda] nvidia-curand-cu12        10.3.5.147               pypi_0    pypi
[conda] nvidia-cusolver-cu12      11.6.1.9                 pypi_0    pypi
[conda] nvidia-cusparse-cu12      12.3.1.170               pypi_0    pypi
[conda] nvidia-nccl-cu12          2.21.5                   pypi_0    pypi
[conda] nvidia-nvjitlink-cu12     12.4.127                 pypi_0    pypi
[conda] nvidia-nvtx-cu12          12.4.127                 pypi_0    pypi
[conda] optree                    0.13.0                   pypi_0    pypi
[conda] pytorch-triton            3.0.0+dedb7bdf33          pypi_0    pypi
[conda] torch                     2.6.0a0+git13eb3b3           dev_0    <develop>
[conda] torchfix                  0.4.0                    pypi_0    pypi
[conda] triton                    3.1.0                    pypi_0    pypi


cc @ezyang @chauhang @penguinwu @voznesenskym @EikanWang @jgong5 @Guobing-Chen @XiaobingSuper @zhuhaozhe @blzheng @wenzhe-nrv @jiayisunx @ipiszy @yf225 @chenyang78 @kadeng @muchulee8 @ColinPeppler @amjames @desertfire @aakhundov