torch._dynamo.exc.Unsupported: call_function BuiltinVariable(zip) [ListVariable(), ListVariable(), ListVariable(), UserDefinedObjectVariable(KJTList)] {}
Labels: triaged, oncall: pt2, module: dynamo

### 🐛 Describe the bug

Steps to reproduce:

pytorch version: https://github.com/pytorch/pytorch/commit/796ce672296c9ae8d7387b18403810aa2f1048a1
torchrec version: 005727ef06cdc808aa7d263ab4f2837938a77ed2

1. Get working torchrec install (you need fbgemm-gpu and torchrec, install by source; explicitly uninstall fbgemm-gpu-nightly and torchrec-nightly if they exist)
2. Patch torchrec with https://gist.github.com/ezyang/e8a60401b52a3c7f630c1f7c072ec9f1
3. For ease of diagnosis, patch pytorch with https://github.com/pytorch/pytorch/pull/107683 (but I don't think it is strictly necessary)
4. Patch

```
diff --git a/torch/_dynamo/config.py b/torch/_dynamo/config.py
index b9e828750c7..17eab15f945 100644
--- a/torch/_dynamo/config.py
+++ b/torch/_dynamo/config.py
@@ -205,7 +205,7 @@ enforce_cond_guards_match = True
 # run without graph-breaks, but also without comm/compute overlap.
 # set torch._dynamo.config.log_level to INFO or DEBUG for more info
 # about optimize_ddp behavior.
-optimize_ddp = True
+optimize_ddp = False
 
 # Whether to skip guarding on FSDP-managed modules
 skip_fsdp_guards = True
```

5. Run TORCH_LOGS=+dynamo MASTER_ADDR=127.0.0.1 MASTER_PORT=29501 RANK=0 LOCAL_RANK=0 WORLD_SIZE=1 python train_dlrm.py

```
Traceback (most recent call last):
  File "/data/users/ezyang/a/torchrec/examples/golden_training/train_dlrm.py", line 164, in <module>
    main()
  File "/data/users/ezyang/a/pytorch/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper
    return f(*args, **kwargs)
  File "/data/users/ezyang/a/torchrec/examples/golden_training/train_dlrm.py", line 52, in main
    train()
  File "/data/users/ezyang/a/torchrec/examples/golden_training/train_dlrm.py", line 157, in train
    print(model(next(train_iterator).to(device)))
  File "/data/users/ezyang/a/pytorch/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/users/ezyang/a/torchrec/torchrec/distributed/model_parallel.py", line 266, in forward
    return self._dmp_wrapped_module(*args, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/nn/parallel/distributed.py", line 1519, in forward
    else self._run_ddp_forward(*inputs, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/nn/parallel/distributed.py", line 1355, in _run_ddp_forward
    return self.module(*inputs, **kwargs)  # type: ignore[index]
  File "/data/users/ezyang/a/pytorch/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/eval_frame.py", line 333, in _fn
    return fn(*args, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/eval_frame.py", line 493, in catch_errors
    return callback(frame, cache_size, hooks, frame_state)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/convert_frame.py", line 133, in _fn
    return fn(*args, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/convert_frame.py", line 379, in _convert_frame_assert
    return _compile(
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/convert_frame.py", line 554, in _compile
    guarded_code = compile_inner(code, one_graph, hooks, transform)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/utils.py", line 181, in time_wrapper
    r = func(*args, **kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/convert_frame.py", line 476, in compile_inner
    out_code = transform_code_object(code, transform)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/bytecode_transformation.py", line 1028, in transform_code_object
    transformations(instructions, code_options)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/convert_frame.py", line 443, in transform
    tracer.run()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2074, in run
    super().run()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 724, in run
    and self.step()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in step
    getattr(self, inst.opname)(inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 392, in wrapper
    return inner_fn(self, inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 1115, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 562, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/nn_module.py", line 331, in call_function
    return tx.inline_user_function_return(
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 598, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2179, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2286, in inline_call_
    tracer.run()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 724, in run
    and self.step()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in step
    getattr(self, inst.opname)(inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 392, in wrapper
    return inner_fn(self, inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 1155, in CALL_FUNCTION_EX
    self.call_function(fn, argsvars.items, kwargsvars.items)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 562, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 307, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 261, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 90, in call_function
    return tx.inline_user_function_return(
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 598, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2179, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2286, in inline_call_
    tracer.run()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 724, in run
    and self.step()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in step
    getattr(self, inst.opname)(inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 392, in wrapper
    return inner_fn(self, inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 1115, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 562, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/nn_module.py", line 331, in call_function
    return tx.inline_user_function_return(
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 598, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2179, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2286, in inline_call_
    tracer.run()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 724, in run
    and self.step()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in step
    getattr(self, inst.opname)(inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 392, in wrapper
    return inner_fn(self, inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 1155, in CALL_FUNCTION_EX
    self.call_function(fn, argsvars.items, kwargsvars.items)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 562, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 307, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 261, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 90, in call_function
    return tx.inline_user_function_return(
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 598, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2179, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2286, in inline_call_
    tracer.run()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 724, in run
    and self.step()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in step
    getattr(self, inst.opname)(inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 392, in wrapper
    return inner_fn(self, inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 1115, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 562, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/nn_module.py", line 716, in call_function
    ).call_function(tx, [self] + list(args), kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 261, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 90, in call_function
    return tx.inline_user_function_return(
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 598, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2179, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2286, in inline_call_
    tracer.run()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 724, in run
    and self.step()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in step
    getattr(self, inst.opname)(inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 392, in wrapper
    return inner_fn(self, inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 1155, in CALL_FUNCTION_EX
    self.call_function(fn, argsvars.items, kwargsvars.items)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 562, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 307, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 261, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 90, in call_function
    return tx.inline_user_function_return(
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 598, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2179, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2286, in inline_call_
    tracer.run()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 724, in run
    and self.step()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in step
    getattr(self, inst.opname)(inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 392, in wrapper
    return inner_fn(self, inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 1115, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 562, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 307, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 261, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/functions.py", line 90, in call_function
    return tx.inline_user_function_return(
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 598, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2179, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 2286, in inline_call_
    tracer.run()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 724, in run
    and self.step()
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in step
    getattr(self, inst.opname)(inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 392, in wrapper
    return inner_fn(self, inst)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 1115, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/symbolic_convert.py", line 562, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/builtin.py", line 635, in call_function
    return super().call_function(tx, args, kwargs)
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/variables/base.py", line 306, in call_function
    unimplemented(f"call_function {self} {args} {kwargs}")
  File "/data/users/ezyang/a/pytorch/torch/_dynamo/exc.py", line 172, in unimplemented
    raise Unsupported(msg)
torch._dynamo.exc.Unsupported: call_function BuiltinVariable(zip) [ListVariable(), ListVariable(), ListVariable(), UserDefinedObjectVariable(KJTList)] {}

from user code:
   File "/data/users/ezyang/a/torchrec/torchrec/models/dlrm.py", line 893, in forward
    logits = self.model(batch.dense_features, batch.sparse_features)
  File "/data/users/ezyang/a/pytorch/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/users/ezyang/a/torchrec/torchrec/models/dlrm.py", line 571, in forward
    embedded_sparse = self.sparse_arch(sparse_features)
  File "/data/users/ezyang/a/pytorch/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/users/ezyang/a/torchrec/torchrec/models/dlrm.py", line 99, in forward
    sparse_features: KeyedTensor = self.embedding_bag_collection(features)
  File "/data/users/ezyang/a/pytorch/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/users/ezyang/a/torchrec/torchrec/distributed/types.py", line 700, in forward
    return self.compute_and_output_dist(ctx, dist_input)
  File "/data/users/ezyang/a/torchrec/torchrec/distributed/embeddingbag.py", line 704, in compute_and_output_dist
    for lookup, dist, sharding_ctx, features in zip(

```

Full debug log https://gist.github.com/ezyang/84f0c66e349682f689f3fb0b38433b0e

### Versions

main

cc @msaroufim @wconstab @bdhirsh @anijain2305 @voznesenskym @penguinwu @EikanWang @jgong5 @Guobing-Chen @XiaobingSuper @zhuhaozhe @blzheng @Xia-Weiwen @wenzhe-nrv @jiayisunx @chenyang78 @aakhundov