Repository: tensorflow
Issue: gradient returns `None`
Labels: stat:awaiting tensorflower,comp:ops,type:performance,TF 2.7

### Issue type

Performance

### Have you reproduced the bug with TensorFlow Nightly?

No

### Source

source

### TensorFlow version

2.7.1

### Custom code

Yes

### OS platform and distribution

CentOS Linux 7 (Core)

### Mobile device

_No response_

### Python version

3.8.12

### Bazel version

_No response_

### GCC/compiler version

_No response_

### CUDA/cuDNN version

_No response_

### GPU model and memory

Running on CPU only

### Current behavior?

In the code below I don't understand why `test_tape` which is a `tf.GradientTape()` returns an empty (`None`) gradient. I have made sure to call the models so weights are initialized, checked that losses are actual tensors, redefined inputs as `tf.Variable()` and alternatively attempted using `tape.watch()`. My intuition is that the first call to `train_optimizer.apply_gradients()` somehow breaks the computational graph which is why `test_tape` does not track variables associated with `model_copy` but I am not sure how to fix the issue.

### Standalone code to reproduce the issue

```shell
import tensorflow as tf
import tensorflow.keras as keras
import tensorflow.keras.backend as keras_backend
import numpy as np

def sineGenerator(amplitude=None, phase=None):
    if amplitude is None:
        amplitude = tf.random.uniform(shape=[], minval=0.1, maxval=5.0)
    if phase is None: 
        phase     = tf.random.uniform(shape=[], minval=0.0, maxval=np.pi)
    def _gen(x):
        return amplitude*tf.math.sin(x+phase)
    return _gen
    
def genX(sample, minval=-5.0, maxval=5.0):
    return tf.expand_dims(tf.random.uniform(shape=[sample], minval=minval, maxval=maxval), 1)

class sineModel(keras.Model):
    def __init__(self):
        super().__init__()
        self.hidden1 = keras.layers.Dense(40, input_shape=(1,))
        self.hidden2 = keras.layers.Dense(40)
        self.out     = keras.layers.Dense(1)
        
    def call(self, x):
        x = keras.activations.relu(self.hidden1(x))
        x = keras.activations.relu(self.hidden2(x))
        x = self.out(x)
        return x
    
def copyModel(model, x):
    model_copy = sineModel()
    output = model_copy(x)
    output = model(x) # should not be necessary since model(train_x) is called before model_copy = copyModel(model,x) in maml_train()
    model_copy.set_weights(model.get_weights())
    return model_copy


def maml_train(model, total_iterations=10, meta_train_steps=10, meta_test_steps=100):
    log_step = total_iterations //10 if total_iterations > 10 else 1
    
    test_optimizer  = keras.optimizers.Adam(learning_rate=0.001)
    train_optimizer = keras.optimizers.Adam(learning_rate=0.001)
    
    losses, total_loss = [], 0.
    
    for step in range(total_iterations):
        sineGen = sineGenerator()
        
        test_x     = genX(meta_test_steps)
        test_y     = sineGen(test_x)
        
        train_x    = genX(meta_train_steps)
        train_y    = sineGen(train_x)
        
#         test_x, test_y, train_x, train_y = tf.Variable(test_x), tf.Variable(test_y), tf.Variable(train_x), tf.Variable(train_y)
#         test_x, test_y, train_x, train_y = tf.convert_to_tensor(test_x), tf.convert_to_tensor(test_y), tf.convert_to_tensor(train_x), tf.convert_to_tensor(train_y)
        
        model(train_x)
        model_copy = copyModel(model, train_x)
        
        
        with tf.GradientTape() as test_tape:
#             test_tape.watch([test_x, test_y, train_x, train_y])
            with tf.GradientTape() as train_tape:
#                 train_tape.watch([test_x, test_y, train_x, train_y])
                train_loss = tf.reduce_mean(keras.losses.mean_squared_error(train_y, model(train_x)))
            
#             print('train_loss type', type(train_loss))
            gradients = train_tape.gradient(train_loss, model.trainable_variables)
            train_optimizer.apply_gradients(zip(gradients, model_copy.trainable_variables))
            
            test_loss = tf.reduce_mean(keras.losses.mean_squared_error(test_y, model_copy(test_x)))
            
#         print('test_loss type', type(test_loss))
        gradients = test_tape.gradient(test_loss, model.trainable_variables)
        print(gradients)
        test_optimizer.apply_gradients(zip(gradients, model.trainable_variables))

        total_loss += test_loss
        losses += [total_loss/(step+1)]
        
        
        if step % log_step == 0:
             print('Training loss (total) at step %d: \t %.4f' % (step, total_loss/(step+1)))
    return losses

sModel_maml = sineModel()
maml_train(sModel_maml)
```


### Relevant log output

```shell
[None, None, None, None, None, None]

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-402-4ed8f16df6b0> in <module>
     76 
     77 sModel_maml = sineModel2()
---> 78 maml_train(sModel_maml)

<ipython-input-402-4ed8f16df6b0> in maml_train(model, total_iterations, meta_train_steps, meta_test_steps)
     65         gradients = test_tape.gradient(test_loss, model.trainable_variables)
     66         print(gradients)
---> 67         test_optimizer.apply_gradients(zip(gradients, model.trainable_variables))
     68 
     69         total_loss += test_loss

~/pyenvs/tensorflow2/lib/python3.8/site-packages/keras/optimizer_v2/optimizer_v2.py in apply_gradients(self, grads_and_vars, name, experimental_aggregate_gradients)
    631       RuntimeError: If called in a cross-replica context.
    632     """
--> 633     grads_and_vars = optimizer_utils.filter_empty_gradients(grads_and_vars)
    634     var_list = [v for (_, v) in grads_and_vars]
    635 

~/pyenvs/tensorflow2/lib/python3.8/site-packages/keras/optimizer_v2/utils.py in filter_empty_gradients(grads_and_vars)
     71   if not filtered:
     72     variable = ([v.name for _, v in grads_and_vars],)
---> 73     raise ValueError(f"No gradients provided for any variable: {variable}. "
     74                      f"Provided `grads_and_vars` is {grads_and_vars}.")
     75   if vars_with_empty_grads:

ValueError: No gradients provided for any variable: (['sine_model2_59/dense_555/kernel:0', 'sine_model2_59/dense_555/bias:0', 'sine_model2_59/dense_556/kernel:0', 'sine_model2_59/dense_556/bias:0', 'sine_model2_59/dense_557/kernel:0', 'sine_model2_59/dense_557/bias:0'],). Provided `grads_and_vars` is ((None, <tf.Variable 'sine_model2_59/dense_555/kernel:0' shape=(1, 40) dtype=float32, numpy=....
```