{
	"name": "BackendCompilerFailed",
	"message": "backend='inductor' raised:
ImportError: cannot import name 'triton_key' from 'triton.compiler.compiler' (/home/developer6/.local/lib/python3.10/site-packages/triton/compiler/compiler.py)

Set TORCH_LOGS=\"+dynamo\" and TORCHDYNAMO_VERBOSE=1 for more information
",
	"stack": "---------------------------------------------------------------------------
BackendCompilerFailed                     Traceback (most recent call last)
Cell In[1], line 3
      1 from fam.llm.ckpt_inference import TTS
----> 3 model = TTS(ckpt_path='/home/developer6/.cache/huggingface/hub/models--kotoba-speech--azure-8node-3cbooks_bpe_mls_resumed_bjkmcu1e/snapshots/17f3f0a85d3380169988382b1fba69e279d5e05f/epoch=8-step=1099726.ckpt')

File ~/kotoba-voice/fam/llm/ckpt_inference.py:65, in TTS.__init__(self, ckpt_path, seed, output_dir, first_model_path)
     62 self.enhancer = get_enhancer(\"df\")
     64 self.precision = {\"float16\": torch.float16, \"bfloat16\": torch.bfloat16}[self._dtype]
---> 65 self.model, self.tokenizer, self.smodel, self.model_size = build_model(
     66     precision=self.precision,
     67     checkpoint_path=Path(ckpt_path),
     68     spk_emb_ckpt_path=Path('/home/developer6/.cache/huggingface/hub/models--kotoba-tech--kotoba-speech-v0.1/snapshots/c8f3ce36fabbf1315a9f3979e9dcdc890c44005a/speaker_encoder.pt'),
     69     device=self._device,
     70     compile=True,
     71     compile_prefill=True,
     72     first_model_path= None,
     73 )
     75 mbd_bandwidth = 6  # 1.5
     76 self.mbd_sample_rate = 24_000

File ~/kotoba-voice/fam/llm/fast_inference_utils.py:415, in build_model(precision, checkpoint_path, spk_emb_ckpt_path, compile_prefill, compile, device, first_model_path)
    413 device_sync(device=device)  # MKG
    414 t0 = time.perf_counter()
--> 415 y, second_tokens = generate(
    416     model,
    417     encoded,
    418     spk_emb,
    419     max_new_tokens=200,
    420     callback=lambda x: x,
    421     temperature=torch.tensor(1.0, device=device, dtype=precision),
    422     top_k=None,
    423     top_p=torch.tensor(0.95, device=device, dtype=precision),
    424     guidance_scale=torch.tensor(3.0, device=device, dtype=precision),
    425     end_of_audio_token=9999,  # don't end early for compilation stage.
    426 )
    428 device_sync(device=device)  # MKG
    430 print(f\"Compilation time: {time.perf_counter() - t0:.2f} seconds\")

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/utils/_contextlib.py:116, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    113 @functools.wraps(func)
    114 def decorate_context(*args, **kwargs):
    115     with ctx_factory():
--> 116         return func(*args, **kwargs)

File ~/kotoba-voice/fam/llm/fast_inference_utils.py:213, in generate(model, prompt, spk_emb, max_new_tokens, callback, end_of_audio_token, **sampling_kwargs)
    210 seq = torch.clone(prompt)
    211 input_pos = torch.arange(0, T, device=device)
--> 213 next_token, second_tokens = prefill(model, prompt.view(1, -1).repeat(2, 1), spk_emb, input_pos, **sampling_kwargs)
    214 seq = torch.cat([seq, next_token.view(1)])
    216 input_pos = torch.tensor([T], device=device, dtype=torch.int)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py:433, in _TorchDynamoContext.__call__.<locals>._fn(*args, **kwargs)
    428 saved_dynamic_layer_stack_depth = (
    429     torch._C._functorch.get_dynamic_layer_stack_depth()
    430 )
    432 try:
--> 433     return fn(*args, **kwargs)
    434 finally:
    435     # Restore the dynamic layer stack depth if necessary.
    436     torch._C._functorch.pop_dynamic_layer_stack_and_undo_to_depth(
    437         saved_dynamic_layer_stack_depth
    438     )

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py:1116, in CatchErrorsWrapper.__call__(self, frame, cache_entry, frame_state)
   1110             return hijacked_callback(
   1111                 frame, cache_entry, self.hooks, frame_state
   1112             )
   1114 with compile_lock, _disable_current_modes():
   1115     # skip=1: skip this frame
-> 1116     return self._torchdynamo_orig_callable(
   1117         frame, cache_entry, self.hooks, frame_state, skip=1
   1118     )

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py:472, in ConvertFrameAssert.__call__(self, frame, cache_entry, hooks, frame_state, skip)
    458 compile_id = CompileId(frame_id, frame_compile_id)
    460 signpost_event(
    461     \"dynamo\",
    462     \"_convert_frame_assert._compile\",
   (...)
    469     },
    470 )
--> 472 return _compile(
    473     frame.f_code,
    474     frame.f_globals,
    475     frame.f_locals,
    476     frame.f_builtins,
    477     self._torchdynamo_orig_callable,
    478     self._one_graph,
    479     self._export,
    480     self._export_constraints,
    481     hooks,
    482     cache_entry,
    483     cache_size,
    484     frame,
    485     frame_state=frame_state,
    486     compile_id=compile_id,
    487     skip=skip + 1,
    488 )

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_utils_internal.py:84, in compile_time_strobelight_meta.<locals>.compile_time_strobelight_meta_inner.<locals>.wrapper_function(*args, **kwargs)
     82 if \"skip\" in kwargs:
     83     kwargs[\"skip\"] = kwargs[\"skip\"] + 1
---> 84 return StrobelightCompileTimeProfiler.profile_compile_time(
     85     function, phase_name, *args, **kwargs
     86 )

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_strobelight/compile_time_profiler.py:129, in StrobelightCompileTimeProfiler.profile_compile_time(cls, func, phase_name, *args, **kwargs)
    124 @classmethod
    125 def profile_compile_time(
    126     cls, func: Any, phase_name: str, *args: Any, **kwargs: Any
    127 ) -> Any:
    128     if not cls.enabled:
--> 129         return func(*args, **kwargs)
    131     if cls.profiler is None:
    132         logger.error(\"profiler is not set\")

File ~/anaconda3/envs/py310/lib/python3.10/contextlib.py:79, in ContextDecorator.__call__.<locals>.inner(*args, **kwds)
     76 @wraps(func)
     77 def inner(*args, **kwds):
     78     with self._recreate_cm():
---> 79         return func(*args, **kwds)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py:817, in _compile(code, globals, locals, builtins, compiler_fn, one_graph, export, export_constraints, hooks, cache_entry, cache_size, frame, frame_state, compile_id, skip)
    815 guarded_code = None
    816 try:
--> 817     guarded_code = compile_inner(code, one_graph, hooks, transform)
    818     return guarded_code
    819 except (
    820     Unsupported,
    821     TorchRuntimeError,
   (...)
    828     BisectValidationException,
    829 ) as e:

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/utils.py:230, in dynamo_timed.<locals>.dynamo_timed_inner.<locals>.time_wrapper(*args, **kwargs)
    228 with torch.profiler.record_function(f\"{key} (dynamo_timed)\"):
    229     t0 = time.time()
--> 230     r = func(*args, **kwargs)
    231     time_spent = time.time() - t0
    232 compilation_time_metrics[key].append(time_spent)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py:636, in _compile.<locals>.compile_inner(code, one_graph, hooks, transform)
    634 CompileContext.get().attempt = attempt
    635 try:
--> 636     out_code = transform_code_object(code, transform)
    637     break
    638 except exc.RestartAnalysis as e:

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py:1185, in transform_code_object(code, transformations, safe)
   1182 instructions = cleaned_instructions(code, safe)
   1183 propagate_line_nums(instructions)
-> 1185 transformations(instructions, code_options)
   1186 return clean_and_assemble_instructions(instructions, keys, code_options)[1]

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py:178, in preserve_global_state.<locals>._fn(*args, **kwargs)
    176 cleanup = setup_compile_debug()
    177 try:
--> 178     return fn(*args, **kwargs)
    179 finally:
    180     cleanup.close()

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py:582, in _compile.<locals>.transform(instructions, code_options)
    580 try:
    581     with tracing(tracer.output.tracing_context), tracer.set_current_tx():
--> 582         tracer.run()
    583 except exc.UnspecializeRestartAnalysis:
    584     speculation_log.clear()

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py:2456, in InstructionTranslator.run(self)
   2455 def run(self):
-> 2456     super().run()

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py:898, in InstructionTranslatorBase.run(self)
    896 try:
    897     self.output.push_tx(self)
--> 898     while self.step():
    899         pass
    900 except BackendCompilerFailed:

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py:810, in InstructionTranslatorBase.step(self)
    807 self.update_block_stack(inst)
    809 try:
--> 810     self.dispatch_table[inst.opcode](self, inst)
    811     return not self.output.should_exit
    812 except exc.ObservedException:

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py:2647, in InstructionTranslator.RETURN_VALUE(self, inst)
   2646 def RETURN_VALUE(self, inst):
-> 2647     self._return(inst)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py:2632, in InstructionTranslator._return(self, inst)
   2627 _step_logger()(
   2628     logging.INFO,
   2629     f\"torchdynamo done tracing {self.f_code.co_name} ({inst.opname})\",
   2630 )
   2631 log.debug(\"%s triggered compile\", inst.opname)
-> 2632 self.output.compile_subgraph(
   2633     self,
   2634     reason=GraphCompileReason(
   2635         \"return_value\", [self.frame_summary()], graph_break=False
   2636     ),
   2637 )
   2638 return_inst = (
   2639     create_instruction(\"RETURN_VALUE\")
   2640     if inst.opname == \"RETURN_VALUE\"
   2641     else create_instruction(\"RETURN_CONST\", argval=inst.argval)
   2642 )
   2643 self.output.add_output_instructions([return_inst])

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/output_graph.py:1123, in OutputGraph.compile_subgraph(self, tx, partial_convert, reason)
   1120 output = []
   1121 if count_calls(self.graph) != 0 or len(pass2.graph_outputs) != 0:
   1122     output.extend(
-> 1123         self.compile_and_call_fx_graph(tx, pass2.graph_output_vars(), root)
   1124     )
   1126     if len(pass2.graph_outputs) != 0:
   1127         output.append(pass2.create_store(graph_output_var))

File ~/anaconda3/envs/py310/lib/python3.10/contextlib.py:79, in ContextDecorator.__call__.<locals>.inner(*args, **kwds)
     76 @wraps(func)
     77 def inner(*args, **kwds):
     78     with self._recreate_cm():
---> 79         return func(*args, **kwds)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/output_graph.py:1318, in OutputGraph.compile_and_call_fx_graph(self, tx, rv, root)
   1315     self.tracing_context.fake_mode = backend_fake_mode
   1317 with self.restore_global_state():
-> 1318     compiled_fn = self.call_user_compiler(gm)
   1320 from torch.fx._lazy_graph_module import _LazyGraphModule
   1322 if isinstance(compiled_fn, _LazyGraphModule) or (
   1323     isinstance(getattr(compiled_fn, \"__self__\", None), _LazyGraphModule)
   1324     and compiled_fn.__name__ == \"_lazy_forward\"
   (...)
   1328     # this is a _LazyGraphModule. This makes it easier for dynamo to
   1329     # optimize a _LazyGraphModule.

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/utils.py:230, in dynamo_timed.<locals>.dynamo_timed_inner.<locals>.time_wrapper(*args, **kwargs)
    228 with torch.profiler.record_function(f\"{key} (dynamo_timed)\"):
    229     t0 = time.time()
--> 230     r = func(*args, **kwargs)
    231     time_spent = time.time() - t0
    232 compilation_time_metrics[key].append(time_spent)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/output_graph.py:1409, in OutputGraph.call_user_compiler(self, gm)
   1407     raise e
   1408 except Exception as e:
-> 1409     raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
   1410         e.__traceback__
   1411     ) from None
   1413 signpost_event(
   1414     \"dynamo\",
   1415     \"OutputGraph.call_user_compiler\",
   (...)
   1421     },
   1422 )
   1424 return compiled_fn

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/output_graph.py:1390, in OutputGraph.call_user_compiler(self, gm)
   1388 if config.verify_correctness:
   1389     compiler_fn = WrapperBackend(compiler_fn)
-> 1390 compiled_fn = compiler_fn(gm, self.example_inputs())
   1391 _step_logger()(logging.INFO, f\"done compiler function {name}\")
   1392 assert callable(compiled_fn), \"compiler_fn did not return callable\"

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/repro/after_dynamo.py:129, in WrapBackendDebug.__call__(self, gm, example_inputs, **kwargs)
    127             raise
    128 else:
--> 129     compiled_gm = compiler_fn(gm, example_inputs)
    131 return compiled_gm

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/repro/after_dynamo.py:129, in WrapBackendDebug.__call__(self, gm, example_inputs, **kwargs)
    127             raise
    128 else:
--> 129     compiled_gm = compiler_fn(gm, example_inputs)
    131 return compiled_gm

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/__init__.py:1951, in _TorchCompileInductorWrapper.__call__(self, model_, inputs_)
   1948 def __call__(self, model_, inputs_):
   1949     from torch._inductor.compile_fx import compile_fx
-> 1951     return compile_fx(model_, inputs_, config_patches=self.config)

File ~/anaconda3/envs/py310/lib/python3.10/contextlib.py:79, in ContextDecorator.__call__.<locals>.inner(*args, **kwds)
     76 @wraps(func)
     77 def inner(*args, **kwds):
     78     with self._recreate_cm():
---> 79         return func(*args, **kwds)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_inductor/compile_fx.py:1504, in compile_fx(model_, example_inputs_, inner_compile, config_patches, decompositions)
   1499         return inference_compiler(unlifted_gm, example_inputs_)
   1501 with V.set_fake_mode(fake_mode), torch._guards.tracing(
   1502     tracing_context
   1503 ), compiled_autograd.disable(), functorch_config.patch(unlift_effect_tokens=True):
-> 1504     return aot_autograd(
   1505         fw_compiler=fw_compiler,
   1506         bw_compiler=bw_compiler,
   1507         inference_compiler=inference_compiler,
   1508         decompositions=decompositions,
   1509         partition_fn=partition_fn,
   1510         keep_inference_input_mutations=True,
   1511     )(model_, example_inputs_)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/backends/common.py:69, in AotAutograd.__call__(self, gm, example_inputs)
     66 try:
     67     # NB: NOT cloned!
     68     with enable_aot_logging(), patch_config:
---> 69         cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
     70         counters[\"aot_autograd\"][\"ok\"] += 1
     71         return disable(cg)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py:953, in aot_module_simplified(mod, args, fw_compiler, bw_compiler, partition_fn, decompositions, keep_inference_input_mutations, inference_compiler)
    937 aot_config = AOTConfig(
    938     fw_compiler=fw_compiler,
    939     bw_compiler=bw_compiler,
   (...)
    949     no_tangents=False,
    950 )
    952 with compiled_autograd.disable():
--> 953     compiled_fn, _ = create_aot_dispatcher_function(
    954         functional_call,
    955         full_args,
    956         aot_config,
    957     )
    959 if isinstance(mod, torch._dynamo.utils.GmWrapper):
    960     # This function is called by the flatten_graph_inputs wrapper, which boxes
    961     # the inputs so that they can be freed before the end of this scope.
    962     # For overhead reasons, this is not the default wrapper, see comment:
    963     # https://github.com/pytorch/pytorch/pull/122535/files#r1560096481
    964     def boxed_forward(runtime_args: List[Any]):

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/utils.py:230, in dynamo_timed.<locals>.dynamo_timed_inner.<locals>.time_wrapper(*args, **kwargs)
    228 with torch.profiler.record_function(f\"{key} (dynamo_timed)\"):
    229     t0 = time.time()
--> 230     r = func(*args, **kwargs)
    231     time_spent = time.time() - t0
    232 compilation_time_metrics[key].append(time_spent)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py:686, in create_aot_dispatcher_function(flat_fn, flat_args, aot_config)
    682         return aot_dispatch_base
    684 compiler_fn = choose_dispatcher(needs_autograd, aot_config)
--> 686 compiled_fn, fw_metadata = compiler_fn(
    687     flat_fn, fake_flat_args, aot_config, fw_metadata=fw_metadata
    688 )
    689 return compiled_fn, fw_metadata

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py:168, in aot_dispatch_base(flat_fn, flat_args, aot_config, fw_metadata)
    161     tracing_context.fw_metadata = (
    162         fw_metadata
    163         if maybe_subclass_meta is None
    164         else maybe_subclass_meta.fw_metadata
    165     )
    167 with TracingContext.report_output_strides() as fwd_output_strides:
--> 168     compiled_fw = compiler(fw_module, updated_flat_args)
    170 if fakified_out_wrapper.needs_post_compile:
    171     fakified_out_wrapper.set_fwd_output_strides(fwd_output_strides)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/utils.py:230, in dynamo_timed.<locals>.dynamo_timed_inner.<locals>.time_wrapper(*args, **kwargs)
    228 with torch.profiler.record_function(f\"{key} (dynamo_timed)\"):
    229     t0 = time.time()
--> 230     r = func(*args, **kwargs)
    231     time_spent = time.time() - t0
    232 compilation_time_metrics[key].append(time_spent)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_inductor/compile_fx.py:1409, in compile_fx.<locals>.fw_compiler_base(model, example_inputs, is_inference)
   1401     assert orig_output_end_idx <= num_model_outputs
   1403     user_visible_outputs = dict.fromkeys(
   1404         n.name
   1405         for n in model_outputs[original_output_start_index:orig_output_end_idx]
   1406         if isinstance(n, torch.fx.Node)
   1407     )
-> 1409 return inner_compile(
   1410     model,
   1411     example_inputs,
   1412     static_input_idxs=get_static_input_idxs(fixed),
   1413     cudagraphs=cudagraphs,
   1414     graph_id=graph_id,
   1415     is_inference=is_inference,
   1416     boxed_forward_device_index=forward_device,
   1417     user_visible_outputs=user_visible_outputs,
   1418 )

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/repro/after_aot.py:84, in wrap_compiler_debug.<locals>.debug_wrapper(gm, example_inputs, **kwargs)
     79 assert config.repro_after in (\"dynamo\", \"aot\", None)
     81 try:
     82     # Call the compiler_fn - which is either aot_autograd or inductor
     83     # with fake inputs
---> 84     inner_compiled_fn = compiler_fn(gm, example_inputs)
     85 except Exception as e:
     86     # TODO: Failures here are troublesome because no real inputs,
     87     # need a different serialization strategy
     88     if config.repro_after == \"aot\":

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_inductor/debug.py:304, in DebugContext.wrap.<locals>.inner(*args, **kwargs)
    301 @functools.wraps(fn)
    302 def inner(*args, **kwargs):
    303     with DebugContext():
--> 304         return fn(*args, **kwargs)

File ~/anaconda3/envs/py310/lib/python3.10/contextlib.py:79, in ContextDecorator.__call__.<locals>.inner(*args, **kwds)
     76 @wraps(func)
     77 def inner(*args, **kwds):
     78     with self._recreate_cm():
---> 79         return func(*args, **kwds)

File ~/anaconda3/envs/py310/lib/python3.10/contextlib.py:79, in ContextDecorator.__call__.<locals>.inner(*args, **kwds)
     76 @wraps(func)
     77 def inner(*args, **kwds):
     78     with self._recreate_cm():
---> 79         return func(*args, **kwds)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_dynamo/utils.py:230, in dynamo_timed.<locals>.dynamo_timed_inner.<locals>.time_wrapper(*args, **kwargs)
    228 with torch.profiler.record_function(f\"{key} (dynamo_timed)\"):
    229     t0 = time.time()
--> 230     r = func(*args, **kwargs)
    231     time_spent = time.time() - t0
    232 compilation_time_metrics[key].append(time_spent)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_inductor/compile_fx.py:517, in compile_fx_inner(gm, example_inputs, cudagraphs, static_input_idxs, is_backward, graph_id, cpp_wrapper, aot_mode, is_inference, boxed_forward_device_index, user_visible_outputs, layout_opt, extern_node_serializer)
    510         if (
    511             isinstance(input, torch.Tensor)
    512             and input.device.type == \"cuda\"
    513             and i in static_input_idxs
    514         ):
    515             input._is_inductor_static = True  # type: ignore[attr-defined]
--> 517     compiled_graph = FxGraphCache.load(
    518         fx_codegen_and_compile,
    519         gm,
    520         example_inputs,
    521         graph_kwargs,
    522         inputs_to_check,
    523         local=config.fx_graph_cache,
    524         remote=fx_graph_remote_cache,
    525     )
    526 else:
    527     compiled_graph = fx_codegen_and_compile(
    528         gm, example_inputs, **graph_kwargs  # type: ignore[arg-type]
    529     )

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_inductor/codecache.py:1017, in FxGraphCache.load(compile_fx_fn, gm, example_inputs, fx_kwargs, inputs_to_check, local, remote)
   1015 try:
   1016     FxGraphCache._check_can_cache(gm)
-> 1017     key = compiled_fx_graph_hash(gm, example_inputs, fx_kwargs, inputs_to_check)
   1019     remote_cache = None
   1020     if remote:

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_inductor/codecache.py:702, in compiled_fx_graph_hash(gm, example_inputs, fx_kwargs, inputs_to_check)
    693 def compiled_fx_graph_hash(
    694     gm: torch.fx.GraphModule,
    695     example_inputs: List[torch.Tensor],
    696     fx_kwargs: Dict[str, Any],
    697     inputs_to_check: Sequence[int],
    698 ) -> str:
    699     \"\"\"
    700     Generate a unique hash of the FX graph for caching.
    701     \"\"\"
--> 702     details = FxGraphHashDetails(gm, example_inputs, fx_kwargs, inputs_to_check)
    703     # The prefix distinguishes among the other kinds of objects we
    704     # cache in this module.
    705     key = \"f\" + FxGraphCachePickler.get_hash(details)

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_inductor/codecache.py:681, in FxGraphHashDetails.__init__(self, gm, example_inputs, fx_kwargs, inputs_to_check)
    679 # Also hash on various system info (including the triton compiler version).
    680 self.torch_version = torch_key()
--> 681 self.system_info = CacheBase.get_system()
    682 self.inductor_config = config.save_config_portable()

File ~/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_inductor/codecache.py:143, in CacheBase.get_system()
    139 @staticmethod
    140 @functools.lru_cache(None)
    141 def get_system() -> Dict[str, Any]:
    142     try:
--> 143         from triton.compiler.compiler import triton_key
    145         # Use triton_key instead of triton.__version__ as the version
    146         # is not updated with each code change
    147         triton_version = triton_key()

BackendCompilerFailed: backend='inductor' raised:
ImportError: cannot import name 'triton_key' from 'triton.compiler.compiler' (/home/developer6/.local/lib/python3.10/site-packages/triton/compiler/compiler.py)

Set TORCH_LOGS=\"+dynamo\" and TORCHDYNAMO_VERBOSE=1 for more information
"
}