Repository: pytorch
Issue: dynamo re-uses incorrect compiled frame when changing requires-gradness of model params
Labels: high priority,triaged,oncall: pt2,module: dynamo,module: guards

Repro:
```
from functools import partial
from typing import Any, Callable, Iterable, Optional, Tuple
from contextlib import nullcontext

import torch
import torch.nn as nn

from torch.autograd.profiler import record_function


def adjust_model(model):
    to_freeze = model.num_iter % 2 == 0
    if to_freeze:
        for param in model.layer2.parameters():
            param.requires_grad = False
    else:
        for param in model.layer2.parameters():
            param.requires_grad = True


class MyModule(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(MyModule, self).__init__()

        self.layer1 = nn.Linear(hidden_size, hidden_size)
        self.layer2 = nn.Linear(hidden_size, hidden_size)

        self.num_iter = 0

    def forward(self, x):
        x = self.layer2(x + self.layer1.bias)

        self.num_iter += 1
        return x


# Set random seed for reproducibility
torch.manual_seed(0)

# Generate synthetic data
input_size = 1024
hidden_size = 1024
output_size = 1
num_samples = 2048

# Features are random floats, and labels are also random floats
features = torch.randn(num_samples, input_size, device='cuda')

model = MyModule(input_size, hidden_size, output_size)
model = model.cuda()

model = torch.compile(model)


from torch.profiler import profile, record_function, ProfilerActivity
activities = [ProfilerActivity.CPU, ProfilerActivity.CUDA, ProfilerActivity.XPU]
with profile(activities=activities) as prof:
    for _ in range(10):
        model.zero_grad(True)
        adjust_model(model)
        res = model(features)
        res.sum().backward()

prof.export_chrome_trace("trace_grad_change_compile_bad2.json")
```

The expected behavior in this repro is that:
* every iteration of the fw/bw, we have frozen or unfrozen layer2's weights
* therefore, the number of matmuls in the backward should flipflop between 1 and 2 (when layer2.weight is frozen, we have a single matmul for the gradient of the activation).


The trace that I get is this, though:
<img width="1408" alt="image" src="https://github.com/user-attachments/assets/96d2d840-4f16-4996-9b1e-d956fb09689a">


<img width="1104" alt="image" src="https://github.com/user-attachments/assets/58341643-0fe9-4755-8fdf-f6ee786d861f">


You can see that in each iteration, we end up with 2 matmuls in the backward, and we are also always re-using compiled frame `0/1` (we should be flip-flopping between `0/0` and `0/1`).

Here is the generated tlparse: https://interncache-all.fbcdn.net/manifold/tlparse_reports/tree/logs/hirsheybar/custom/index.html

And here is the logs with `TORCH_LOGS="+guards"`:
```
DEBUG: GUARDS:
DEBUG:
TREE_GUARD_MANAGER:
+- RootGuardManager
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:470 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state()
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- GuardManager: source=L['x'], accessed_by=DictGetItemGuardAccessor(x)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CUDA, BackendSelect, ADInplaceOrView, AutogradCUDA), torch.float32, device=0, requires_grad=False, size=[2048, 1024], stride=[1024, 1])  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | +- NO_TENSOR_ALIASING: check_no_aliasing(L['x'], L['self']._modules['layer1']._parameters['bias'], L['self']._modules['layer2']._parameters['bias'], L['self']._modules['layer2']._parameters['weight'])
| +- GuardManager: source=L['self'], accessed_by=DictGetItemGuardAccessor(self)
| | +- TYPE_MATCH: ___check_type_id(L['self'], 99617504)                         # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | +- GuardManager: source=L['self'].__dict__, accessed_by=GetGenericDictGuardAccessor
| | | +- GuardManager: source=L['self']._buffers, accessed_by=DictGetItemGuardAccessor(_buffers)
| | | | +- DICT_LENGTH: not L['self']._buffers                                        # buffers = self.__dict__.get("_buffers")  # nn/modules/module.py:1994 in __setattr__
| | | +- GuardManager: source=L['self']._modules, accessed_by=DictGetItemGuardAccessor(_modules)
| | | | +- DICT_LENGTH: len(L['self']._modules) == 2                                  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | +- GuardManager: source=L['self']._modules['layer1'], accessed_by=DictGetItemGuardAccessor(layer1)
| | | | | +- TYPE_MATCH: ___check_type_id(L['self']._modules['layer1'], 81972256)      # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | +- GuardManager: source=L['self']._modules['layer1'].__dict__, accessed_by=GetGenericDictGuardAccessor
| | | | | | +- GuardManager: source=L['self']._modules['layer1']._parameters, accessed_by=DictGetItemGuardAccessor(_parameters)
| | | | | | | +- DICT_LENGTH: len(L['self']._modules['layer1']._parameters) == 2            # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | | | +- GuardManager: source=L['self']._modules['layer1']._parameters['weight'], accessed_by=DictGetItemGuardAccessor(weight)
| | | | | | | +- GuardManager: source=L['self']._modules['layer1']._parameters['bias'], accessed_by=DictGetItemGuardAccessor(bias)
| | | | | | | | +- TENSOR_MATCH: check_tensor(L['self']._modules['layer1']._parameters['bias'], Parameter, DispatchKeySet(CUDA, BackendSelect, ADInplaceOrView, AutogradCUDA), torch.float32, device=0, requires_grad=True, size=[1024], stride=[1])  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | | | | +- NO_TENSOR_ALIASING
| | | | +- GuardManager: source=L['self']._modules['layer2'], accessed_by=DictGetItemGuardAccessor(layer2)
| | | | | +- TYPE_MATCH: ___check_type_id(L['self']._modules['layer2'], 81972256)      # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | +- GuardManager: source=L['self']._modules['layer2'].__dict__, accessed_by=GetGenericDictGuardAccessor
| | | | | | +- DICT_CONTAINS: not ___dict_contains('forward', L['self']._modules['layer2'].__dict__)  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | | +- GuardManager: source=L['self']._modules['layer2']._parameters, accessed_by=DictGetItemGuardAccessor(_parameters)
| | | | | | | +- DICT_LENGTH: len(L['self']._modules['layer2']._parameters) == 2            # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | | | | | +- GuardManager: source=L['self']._modules['layer2']._parameters['weight'], accessed_by=DictGetItemGuardAccessor(weight)
| | | | | | | | +- TENSOR_MATCH: check_tensor(L['self']._modules['layer2']._parameters['weight'], Parameter, DispatchKeySet(CUDA, BackendSelect, ADInplaceOrView, AutogradCUDA), torch.float32, device=0, requires_grad=False, size=[1024, 1024], stride=[1024, 1])  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | | | | | | +- NO_TENSOR_ALIASING
| | | | | | | +- GuardManager: source=L['self']._modules['layer2']._parameters['bias'], accessed_by=DictGetItemGuardAccessor(bias)
| | | | | | | | +- TENSOR_MATCH: check_tensor(L['self']._modules['layer2']._parameters['bias'], Parameter, DispatchKeySet(CUDA, BackendSelect, ADInplaceOrView, AutogradCUDA), torch.float32, device=0, requires_grad=False, size=[1024], stride=[1])  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | | | | | | +- NO_TENSOR_ALIASING
| | | +- GuardManager: source=L['self'].num_iter, accessed_by=DictGetItemGuardAccessor(num_iter)
| | | | +- EQUALS_MATCH: L['self'].num_iter == 0                                       # self.num_iter += 1  # tmp4.py:33 in forward
| | | +- GuardManager: source=L['self']._parameters, accessed_by=DictGetItemGuardAccessor(_parameters)
| | | | +- DICT_LENGTH: not L['self']._parameters                                     # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| +- GuardManager: source=G, accessed_by=GlobalsGuardAccessor
| | +- GuardManager: source=G['__builtins_dict___0'], accessed_by=DictGetItemGuardAccessor(__builtins_dict___0)
| | | +- GuardManager: source=G['__builtins_dict___0']['super'], accessed_by=DictGetItemGuardAccessor(super)
| | | | +- ID_MATCH: ___check_obj_id(G['__builtins_dict___0']['super'], 7614144)   # super().__setattr__(name, value)  # nn/modules/module.py:2032 in __setattr__
| | | +- GuardManager: source=G['__builtins_dict___0']['isinstance'], accessed_by=DictGetItemGuardAccessor(isinstance)
| | | | +- ID_MATCH: ___check_obj_id(G['__builtins_dict___0']['isinstance'], 140428831199440)  # if isinstance(value, Parameter):  # nn/modules/module.py:1945 in __setattr__
| | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_linear'], accessed_by=DictGetItemGuardAccessor(__import_torch_dot_nn_dot_modules_dot_linear)
| | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_linear'], 140423965322720)  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_linear'].F, accessed_by=GetAttrGuardAccessor(F)
| | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_linear'].F, 140423965324880)  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_linear'].F.linear, accessed_by=GetAttrGuardAccessor(linear)
| | | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_linear'].F.linear, 140426763478864)  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module'], accessed_by=DictGetItemGuardAccessor(__import_torch_dot_nn_dot_modules_dot_module)
| | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_module'], 140426663046384)  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module'].Buffer, accessed_by=GetAttrGuardAccessor(Buffer)
| | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_module'].Buffer, 79489936)  # if isinstance(value, Buffer) or buffers is not None and name in buffers:  # nn/modules/module.py:1995 in __setattr__
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module'].Module, accessed_by=GetAttrGuardAccessor(Module)
| | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_module'].Module, 80429408)  # if isinstance(value, Module):  # nn/modules/module.py:1966 in __setattr__
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module'].Parameter, accessed_by=GetAttrGuardAccessor(Parameter)
| | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_module'].Parameter, 79484096)  # if isinstance(value, Parameter):  # nn/modules/module.py:1945 in __setattr__
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module']._global_forward_hooks, accessed_by=GetAttrGuardAccessor(_global_forward_hooks)
| | | | +- DICT_LENGTH: not G['__import_torch_dot_nn_dot_modules_dot_module']._global_forward_hooks  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module']._global_backward_hooks, accessed_by=GetAttrGuardAccessor(_global_backward_hooks)
| | | | +- DICT_LENGTH: not G['__import_torch_dot_nn_dot_modules_dot_module']._global_backward_hooks  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module']._global_forward_pre_hooks, accessed_by=GetAttrGuardAccessor(_global_forward_pre_hooks)
| | | | +- DICT_LENGTH: not G['__import_torch_dot_nn_dot_modules_dot_module']._global_forward_pre_hooks  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module']._global_backward_pre_hooks, accessed_by=GetAttrGuardAccessor(_global_backward_pre_hooks)
| | | | +- DICT_LENGTH: not G['__import_torch_dot_nn_dot_modules_dot_module']._global_backward_pre_hooks  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward

DEBUG: GUARDS:
DEBUG:
TREE_GUARD_MANAGER:
+- RootGuardManager
| +- DEFAULT_DEVICE: utils_device.CURRENT_DEVICE == None                           # _dynamo/output_graph.py:470 in init_ambient_guards
| +- GLOBAL_STATE: ___check_global_state()
| +- TORCH_FUNCTION_MODE_STACK: ___check_torch_function_mode_stack()
| +- GuardManager: source=L['x'], accessed_by=DictGetItemGuardAccessor(x)
| | +- TENSOR_MATCH: check_tensor(L['x'], Tensor, DispatchKeySet(CUDA, BackendSelect, ADInplaceOrView, AutogradCUDA), torch.float32, device=0, requires_grad=False, size=[2048, 1024], stride=[1024, 1])  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | +- NO_HASATTR: hasattr(L['x'], '_dynamo_dynamic_indices') == False           # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | +- NO_TENSOR_ALIASING: check_no_aliasing(L['x'], L['self']._modules['layer1']._parameters['bias'], L['self']._modules['layer2']._parameters['bias'], L['self']._modules['layer2']._parameters['weight'])
| +- GuardManager: source=L['self'], accessed_by=DictGetItemGuardAccessor(self)
| | +- TYPE_MATCH: ___check_type_id(L['self'], 99617504)                         # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | +- GuardManager: source=L['self'].__dict__, accessed_by=GetGenericDictGuardAccessor
| | | +- GuardManager: source=L['self']._buffers, accessed_by=DictGetItemGuardAccessor(_buffers)
| | | | +- DICT_LENGTH: not L['self']._buffers                                        # buffers = self.__dict__.get("_buffers")  # nn/modules/module.py:1994 in __setattr__
| | | +- GuardManager: source=L['self']._modules, accessed_by=DictGetItemGuardAccessor(_modules)
| | | | +- DICT_LENGTH: len(L['self']._modules) == 2                                  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | +- GuardManager: source=L['self']._modules['layer1'], accessed_by=DictGetItemGuardAccessor(layer1)
| | | | | +- TYPE_MATCH: ___check_type_id(L['self']._modules['layer1'], 81972256)      # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | +- GuardManager: source=L['self']._modules['layer1'].__dict__, accessed_by=GetGenericDictGuardAccessor
| | | | | | +- GuardManager: source=L['self']._modules['layer1']._parameters, accessed_by=DictGetItemGuardAccessor(_parameters)
| | | | | | | +- DICT_LENGTH: len(L['self']._modules['layer1']._parameters) == 2            # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | | | +- GuardManager: source=L['self']._modules['layer1']._parameters['weight'], accessed_by=DictGetItemGuardAccessor(weight)
| | | | | | | +- GuardManager: source=L['self']._modules['layer1']._parameters['bias'], accessed_by=DictGetItemGuardAccessor(bias)
| | | | | | | | +- TENSOR_MATCH: check_tensor(L['self']._modules['layer1']._parameters['bias'], Parameter, DispatchKeySet(CUDA, BackendSelect, ADInplaceOrView, AutogradCUDA), torch.float32, device=0, requires_grad=True, size=[1024], stride=[1])  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | | | | +- NO_TENSOR_ALIASING
| | | | +- GuardManager: source=L['self']._modules['layer2'], accessed_by=DictGetItemGuardAccessor(layer2)
| | | | | +- TYPE_MATCH: ___check_type_id(L['self']._modules['layer2'], 81972256)      # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | +- GuardManager: source=L['self']._modules['layer2'].__dict__, accessed_by=GetGenericDictGuardAccessor
| | | | | | +- DICT_CONTAINS: not ___dict_contains('forward', L['self']._modules['layer2'].__dict__)  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | | | | +- GuardManager: source=L['self']._modules['layer2']._parameters, accessed_by=DictGetItemGuardAccessor(_parameters)
| | | | | | | +- DICT_LENGTH: len(L['self']._modules['layer2']._parameters) == 2            # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | | | | | +- GuardManager: source=L['self']._modules['layer2']._parameters['weight'], accessed_by=DictGetItemGuardAccessor(weight)
| | | | | | | | +- TENSOR_MATCH: check_tensor(L['self']._modules['layer2']._parameters['weight'], Parameter, DispatchKeySet(CUDA, BackendSelect, ADInplaceOrView, AutogradCUDA), torch.float32, device=0, requires_grad=True, size=[1024, 1024], stride=[1024, 1])  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | | | | | | +- NO_TENSOR_ALIASING
| | | | | | | +- GuardManager: source=L['self']._modules['layer2']._parameters['bias'], accessed_by=DictGetItemGuardAccessor(bias)
| | | | | | | | +- TENSOR_MATCH: check_tensor(L['self']._modules['layer2']._parameters['bias'], Parameter, DispatchKeySet(CUDA, BackendSelect, ADInplaceOrView, AutogradCUDA), torch.float32, device=0, requires_grad=True, size=[1024], stride=[1])  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | | | | | | +- NO_TENSOR_ALIASING
| | | +- GuardManager: source=L['self'].num_iter, accessed_by=DictGetItemGuardAccessor(num_iter)
| | | | +- TYPE_MATCH: ___check_type_id(L['self'].num_iter, 7644512)                 # self.num_iter += 1  # tmp4.py:33 in forward
| | | +- GuardManager: source=L['self']._parameters, accessed_by=DictGetItemGuardAccessor(_parameters)
| | | | +- DICT_LENGTH: not L['self']._parameters                                     # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| +- GuardManager: source=G, accessed_by=GlobalsGuardAccessor
| | +- GuardManager: source=G['__builtins_dict___2'], accessed_by=DictGetItemGuardAccessor(__builtins_dict___2)
| | | +- GuardManager: source=G['__builtins_dict___2']['super'], accessed_by=DictGetItemGuardAccessor(super)
| | | | +- ID_MATCH: ___check_obj_id(G['__builtins_dict___2']['super'], 7614144)   # super().__setattr__(name, value)  # nn/modules/module.py:2032 in __setattr__
| | | +- GuardManager: source=G['__builtins_dict___2']['isinstance'], accessed_by=DictGetItemGuardAccessor(isinstance)
| | | | +- ID_MATCH: ___check_obj_id(G['__builtins_dict___2']['isinstance'], 140428831199440)  # if isinstance(value, Parameter):  # nn/modules/module.py:1945 in __setattr__
| | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_linear'], accessed_by=DictGetItemGuardAccessor(__import_torch_dot_nn_dot_modules_dot_linear)
| | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_linear'], 140423965322720)  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_linear'].F, accessed_by=GetAttrGuardAccessor(F)
| | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_linear'].F, 140423965324880)  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_linear'].F.linear, accessed_by=GetAttrGuardAccessor(linear)
| | | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_linear'].F.linear, 140426763478864)  # return F.linear(input, self.weight, self.bias)  # nn/modules/linear.py:125 in forward
| | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module'], accessed_by=DictGetItemGuardAccessor(__import_torch_dot_nn_dot_modules_dot_module)
| | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_module'], 140426663046384)  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module'].Buffer, accessed_by=GetAttrGuardAccessor(Buffer)
| | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_module'].Buffer, 79489936)  # if isinstance(value, Buffer) or buffers is not None and name in buffers:  # nn/modules/module.py:1995 in __setattr__
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module'].Module, accessed_by=GetAttrGuardAccessor(Module)
| | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_module'].Module, 80429408)  # if isinstance(value, Module):  # nn/modules/module.py:1966 in __setattr__
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module'].Parameter, accessed_by=GetAttrGuardAccessor(Parameter)
| | | | +- ID_MATCH: ___check_obj_id(G['__import_torch_dot_nn_dot_modules_dot_module'].Parameter, 79484096)  # if isinstance(value, Parameter):  # nn/modules/module.py:1945 in __setattr__
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module']._global_forward_hooks, accessed_by=GetAttrGuardAccessor(_global_forward_hooks)
| | | | +- DICT_LENGTH: not G['__import_torch_dot_nn_dot_modules_dot_module']._global_forward_hooks  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module']._global_backward_hooks, accessed_by=GetAttrGuardAccessor(_global_backward_hooks)
| | | | +- DICT_LENGTH: not G['__import_torch_dot_nn_dot_modules_dot_module']._global_backward_hooks  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module']._global_forward_pre_hooks, accessed_by=GetAttrGuardAccessor(_global_forward_pre_hooks)
| | | | +- DICT_LENGTH: not G['__import_torch_dot_nn_dot_modules_dot_module']._global_forward_pre_hooks  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward
| | | +- GuardManager: source=G['__import_torch_dot_nn_dot_modules_dot_module']._global_backward_pre_hooks, accessed_by=GetAttrGuardAccessor(_global_backward_pre_hooks)
| | | | +- DICT_LENGTH: not G['__import_torch_dot_nn_dot_modules_dot_module']._global_backward_pre_hooks  # x = self.layer2(x + self.layer1.bias)  # tmp4.py:31 in forward

```

It appears that we are properly emitting guards in the requires_gradness of the model params in both frames (`0/0` and `0/1`), but we are incorrectly dispatching to `0/1` repeatedly.

cc @ezyang @gchanan @zou3519 @kadeng @msaroufim @chauhang @penguinwu @voznesenskym @EikanWang @jgong5 @Guobing-Chen @XiaobingSuper @zhuhaozhe @blzheng @wenzhe-nrv @jiayisunx @chenyang78 @amjames