Repository: GPflow/GPflow

Issue:
Error in gpflow.optimizers.NaturalGradient._natgrad_steps()
<!-- Lines like this are comments and will be invisible -->

## Short summary

I believe the reference to `self._name` in the `NaturalGradient._natgrad_steps()` method should be changed to `self.name`  

<!-- A clear and concise description of what the bug is. -->

## An example where the error shows up

I was working through the [Heteroskedastic Likelihood and Multi-Latent GP](https://gpflow.github.io/GPflow/2.7.1/notebooks/advanced/heteroskedastic.htmll) example in the latest version of the documentation and ran into the following error when calling `optimisation_step()`

```
AttributeError: 'NaturalGradient' object has no attribute '_name'
```

Based on [this S.O. post](https://stackoverflow.com/questions/75778164/gpflow-naturalgradient-object-has-no-attribute-name), at least one other user has encountered the same issue. **I found that the above change to `gpflow/optimizers/natgrad.py` resolved the issue.**


**The code from that example**
<!-- We need to be able to reproduce the bug by simply copy and pasting your code -->
```python
# [1] Import packages
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import tensorflow_probability as tfp

import gpflow as gpf

# [2]: Generate data to be modeled
N = 1001

np.random.seed(0)
tf.random.set_seed(0)

# Build inputs X
X = np.linspace(0, 4 * np.pi, N)[:, None]  # X must be of shape [N, 1]

# Deterministic functions in place of latent ones
f1 = np.sin
f2 = np.cos

# Use transform = exp to ensure positive-only scale values
transform = np.exp

# Compute loc and scale as functions of input X
loc = f1(X)
scale = transform(f2(X))

# Sample outputs Y from Gaussian Likelihood
Y = np.random.normal(loc, scale)


# [3]: Plot the data generated above to see what they look like
def plot_distribution(X, Y, loc, scale):
    plt.figure(figsize=(15, 5))
    x = X.squeeze()
    for k in (1, 2):
        lb = (loc - k * scale).squeeze()
        ub = (loc + k * scale).squeeze()
        plt.fill_between(x, lb, ub, color="silver", alpha=1 - 0.05 * k ** 3)
    plt.plot(x, lb, color="silver")
    plt.plot(x, ub, color="silver")
    plt.plot(X, loc, color="black")
    plt.scatter(X, Y, color="gray", alpha=0.8)
    plt.show()
    plt.close()


#plot_distribution(X, Y, loc, scale)

# [4]: Set up the likelihood function for the model we want
likelihood = gpf.likelihoods.HeteroskedasticTFPConditional(
    distribution_class=tfp.distributions.Normal,  # Gaussian Likelihood
    scale_transform=tfp.bijectors.Exp(),  # Exponential Transform
)

print(f"Likelihood's expected latent_dim: {likelihood.latent_dim}")

# [5]: Specify the kernels of the mean and variance GPs
kernel = gpf.kernels.SeparateIndependent(
    [
        gpf.kernels.SquaredExponential(),  # This is k1, the kernel of f1
        gpf.kernels.SquaredExponential(),  # this is k2, the kernel of f2
    ]
)

# [6]: We will use the SVGP model which requires inducing points
M = 20  # Number of inducing variables for each f_i

# Initial inducing points position Z
Z = np.linspace(X.min(), X.max(), M)[:, None]  # Z must be of shape [M, 1]

inducing_variable = gpf.inducing_variables.SeparateIndependentInducingVariables(
    [
        gpf.inducing_variables.InducingPoints(Z),  # This is U1 = f1(Z1)
        gpf.inducing_variables.InducingPoints(Z),  # This is U2 = f2(Z2)
    ]
)

# [7]: Build the SVGP model using the components instantiated above
model = gpf.models.SVGP(
    kernel=kernel,
    likelihood=likelihood,
    inducing_variable=inducing_variable,
    num_latent_gps=likelihood.latent_dim,
)

#model

# [8]: Build optimizers (NatGrad + Adam)
data = (X, Y)
loss_fn = model.training_loss_closure(data)

gpf.utilities.set_trainable(model.q_mu, False)
gpf.utilities.set_trainable(model.q_sqrt, False)

variational_vars = [(model.q_mu, model.q_sqrt)]
natgrad_opt = gpf.optimizers.NaturalGradient(gamma=0.1)

adam_vars = model.trainable_variables
adam_opt = tf.optimizers.Adam(0.01)


@tf.function
def optimisation_step():
    natgrad_opt.minimize(loss_fn, variational_vars)
    adam_opt.minimize(loss_fn, adam_vars)


# [9]: Run optimization loop
epochs = 100
log_freq = 20

for epoch in range(1, epochs + 1):
    optimisation_step()  # Here's where the error occurs

    # For every 'log_freq' epochs, print the epoch and plot the predictions against the data
    if epoch % log_freq == 0 and epoch > 0:
        print(f"Epoch {epoch} - Loss: {loss_fn().numpy() : .4f}")
        Ymean, Yvar = model.predict_y(X)
        Ymean = Ymean.numpy().squeeze()
        Ystd = tf.sqrt(Yvar).numpy().squeeze()
        plot_distribution(X, Y, Ymean, Ystd)
```

**Error traceback**

I get the following error message using the current version of gpflow:

```
Traceback (most recent call last):
  File "C:\Users\scw8734\Desktop\gp_modeling\gpflow_heteroskedastic.py", line 116, in <module>
    optimisation_step()
  File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\tensorflow\python\util\traceback_utils.py", line 153, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filea1197q1y.py", line 8, in tf__optimisation_step
    ag__.converted_call(ag__.ld(natgrad_opt).minimize, (ag__.ld(loss_fn), ag__.ld(variational_vars)), None, fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filem1g113m3.py", line 13, in tf__wrapped_method
    retval_ = ag__.converted_call(ag__.ld(wrapped_function), (ag__.ld(self),) + tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filegh3cmqhi.py", line 171, in tf__wrapped_function
    ag__.if_stmt(ag__.not_(ag__.converted_call(ag__.ld(get_enable_check_shapes), (), None, fscope)), if_body_4, else_body_4, get_state_7, set_state_7, ('arg_map', 'arg_map[RESULT_TOKEN]', 'checker', 'do_return', 'retval_'), 5)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filegh3cmqhi.py", line 28, in if_body_4
    retval_ = ag__.converted_call(ag__.ld(func), tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filel0u9lm6w.py", line 28, in tf__minimize
    ag__.converted_call(ag__.ld(self)._natgrad_steps, (ag__.ld(loss_fn), ag__.ld(parameters)), None, fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filem1g113m3.py", line 13, in tf__wrapped_method
    retval_ = ag__.converted_call(ag__.ld(wrapped_function), (ag__.ld(self),) + tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filegh3cmqhi.py", line 171, in tf__wrapped_function
    ag__.if_stmt(ag__.not_(ag__.converted_call(ag__.ld(get_enable_check_shapes), (), None, fscope)), if_body_4, else_body_4, get_state_7, set_state_7, ('arg_map', 'arg_map[RESULT_TOKEN]', 'checker', 'do_return', 'retval_'), 5)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filegh3cmqhi.py", line 28, in if_body_4
    retval_ = ag__.converted_call(ag__.ld(func), tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filevzp834lq.py", line 23, in tf___natgrad_steps
    with ag__.ld(tf).name_scope(f'{ag__.ld(self)._name}/natural_gradient_steps'):
AttributeError: in user code:

    File "C:\Users\scw8734\Desktop\gp_modeling\gpflow_heteroskedastic.py", line 107, in optimisation_step  *
        natgrad_opt.minimize(loss_fn, variational_vars)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\check_shapes\integration\tf.py", line 208, in wrapped_method  *
        return wrapped_function(self, *args, **kwargs)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\check_shapes\decorator.py", line 120, in wrapped_function  *
        return func(*args, **kwargs)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\gpflow\optimizers\natgrad.py", line 236, in minimize  *
        self._natgrad_steps(loss_fn, parameters)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\check_shapes\integration\tf.py", line 208, in wrapped_method  *
        return wrapped_function(self, *args, **kwargs)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\check_shapes\decorator.py", line 120, in wrapped_function  *
        return func(*args, **kwargs)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\gpflow\optimizers\natgrad.py", line 266, in _natgrad_steps  *
        with tf.name_scope(f"{self._name}/natural_gradient_steps"):

    AttributeError: 'NaturalGradient' object has no attribute '_name'
```

## Expected behavior

<!-- A clear and concise description of what you expected to happen. -->

The above code should run without error, generating multiple plots over the course of the for loop with output to the console like this:
```
Epoch 20 - Loss:  1471.8678
Epoch 40 - Loss:  1452.4426
Epoch 60 - Loss:  1450.7381
Epoch 80 - Loss:  1450.0566
Epoch 100 - Loss:  1449.6073
```
 
## System information

Package versions:
* matplotlib: 3.7.1
* numpy: 1.23.5
* tensorflow: 2.12.0
* tensorflow_probability: 0.19.0
* gpflow: 2.7.1

**Please let me know if this solution is not appropriate! I'm still just learning how to use gpflow (and getting refreshed on Python).**



Discussion:
This should be resolved by #2050
Ah, wonderful! Should I mark this as closed, or will that happen when #2050 is merged?

Resolving patch:
diff --git a/doc/sphinx/notebooks/advanced/natural_gradients.pct.py b/doc/sphinx/notebooks/advanced/natural_gradients.pct.py
--- a/doc/sphinx/notebooks/advanced/natural_gradients.pct.py
+++ b/doc/sphinx/notebooks/advanced/natural_gradients.pct.py
@@ -22,6 +22,12 @@
 import numpy as np
 import tensorflow as tf
 
+try:
+    # use legacy Adam optimizer to support old TF versions
+    from tensorflow.keras.optimizers.legacy import Adam
+except ImportError:
+    from tensorflow.keras.optimizers import Adam
+
 import gpflow
 from gpflow import set_trainable
 from gpflow.ci_utils import reduce_in_tests
@@ -115,8 +121,8 @@
 set_trainable(vgp.q_mu, False)
 set_trainable(vgp.q_sqrt, False)
 
-adam_opt_for_vgp = tf.optimizers.Adam(adam_learning_rate)
-adam_opt_for_gpr = tf.optimizers.Adam(adam_learning_rate)
+adam_opt_for_vgp = Adam(adam_learning_rate)
+adam_opt_for_gpr = Adam(adam_learning_rate)
 
 # %%
 for i in range(iterations):
@@ -243,7 +249,7 @@
 )
 
 # ordinary gradients with Adam for SVGP
-ordinary_adam_opt = tf.optimizers.Adam(adam_learning_rate)
+ordinary_adam_opt = Adam(adam_learning_rate)
 
 # NatGrads and Adam for SVGP
 # Stop Adam from optimizing the variational parameters
@@ -251,7 +257,7 @@
 set_trainable(svgp_natgrad.q_sqrt, False)
 
 # Create the optimize_tensors for SVGP
-natgrad_adam_opt = tf.optimizers.Adam(adam_learning_rate)
+natgrad_adam_opt = Adam(adam_learning_rate)
 
 natgrad_opt = NaturalGradient(gamma=0.1)
 variational_params = [(svgp_natgrad.q_mu, svgp_natgrad.q_sqrt)]
@@ -333,7 +339,7 @@
 )
 
 # ordinary gradients with Adam for VGP with Bernoulli likelihood
-adam_opt = tf.optimizers.Adam(adam_learning_rate)
+adam_opt = Adam(adam_learning_rate)
 
 # NatGrads and Adam for VGP with Bernoulli likelihood
 # Stop Adam from optimizing the variational parameters
@@ -341,7 +347,7 @@
 set_trainable(vgp_bernoulli_natgrad.q_sqrt, False)
 
 # Create the optimize_tensors for VGP with natural gradients
-natgrad_adam_opt = tf.optimizers.Adam(adam_learning_rate)
+natgrad_adam_opt = Adam(adam_learning_rate)
 natgrad_opt = NaturalGradient(gamma=0.1)
 variational_params = [
     (vgp_bernoulli_natgrad.q_mu, vgp_bernoulli_natgrad.q_sqrt)
@@ -392,7 +398,7 @@
 set_trainable(vgp_bernoulli_natgrads_xi.q_sqrt, False)
 
 # Create the optimize_tensors for VGP with Bernoulli likelihood
-adam_opt = tf.optimizers.Adam(adam_learning_rate)
+adam_opt = Adam(adam_learning_rate)
 natgrad_opt = NaturalGradient(gamma=0.01)
 
 variational_params = [
diff --git a/doc/sphinx/notebooks/advanced/variational_fourier_features.pct.py b/doc/sphinx/notebooks/advanced/variational_fourier_features.pct.py
--- a/doc/sphinx/notebooks/advanced/variational_fourier_features.pct.py
+++ b/doc/sphinx/notebooks/advanced/variational_fourier_features.pct.py
@@ -314,11 +314,11 @@ def _conditional_fused(self, Xnew, full_cov, full_output_cov):
         fmean = tf.matmul(A, f, transpose_a=True)
 
         if q_sqrt is not None:
-            if q_sqrt.get_shape().ndims == 2:
+            if q_sqrt.shape.ndims == 2:
                 # LTA = A * tf.expand_dims(q_sqrt, 2)  # K x M x N
                 # won't work  # make ticket for this?
                 raise NotImplementedError
-            elif q_sqrt.get_shape().ndims == 3:
+            elif q_sqrt.shape.ndims == 3:
                 # L = tf.matrix_band_part(tf.transpose(q_sqrt, (2, 0, 1)), -1, 0)  # K x M x M
 
                 # K x M x N
@@ -331,8 +331,7 @@ def _conditional_fused(self, Xnew, full_cov, full_output_cov):
                 ATL = tf.matmul(A, q_sqrt, transpose_a=True)
             else:
                 raise ValueError(
-                    "Bad dimension for q_sqrt: %s"
-                    % str(q_sqrt.get_shape().ndims)
+                    "Bad dimension for q_sqrt: %s" % str(q_sqrt.shape.ndims)
                 )
             if full_cov:
                 # fvar = fvar + tf.matmul(LTA, LTA, transpose_a=True)  # K x N x N
diff --git a/gpflow/optimizers/natgrad.py b/gpflow/optimizers/natgrad.py
--- a/gpflow/optimizers/natgrad.py
+++ b/gpflow/optimizers/natgrad.py
@@ -201,6 +201,8 @@ def __init__(
         """
         name = self.__class__.__name__ if name is None else name
         super().__init__(name)
+        # explicitly store name (as TF <2.12 stores it differently from TF 2.12+)
+        self.natgrad_name = name
         self.gamma = gamma
         self.xi_transform = xi_transform
 
@@ -263,7 +265,7 @@ def _natgrad_steps(
         q_mu_grads, q_sqrt_grads = tape.gradient(loss, [q_mu_vars, q_sqrt_vars])
         # NOTE that these are the gradients in *unconstrained* space
 
-        with tf.name_scope(f"{self._name}/natural_gradient_steps"):
+        with tf.name_scope(f"{self.natgrad_name}/natural_gradient_steps"):
             for q_mu_grad, q_sqrt_grad, q_mu, q_sqrt, xi_transform in zip(
                 q_mu_grads, q_sqrt_grads, q_mus, q_sqrts, xis
             ):


Test patch: