Repository: GPflow/GPflow

Issue:
How to copy sets of parameter values across models
I would like to be able to copy values of hyperparameters between models, e.g. get a list "all parameters contained within model `m1`", be able to remove some of these (e.g. filter out inducing point locations), then collectively set the corresponding parameters in a different model `m2` to the values gathered from `m1`. There is no such feature at the moment, and e.g. the `understanding/upper_bound.ipynb` notebook contains a rather hacky workaround to achieve this behaviour. It would be neat to have a generally usable utility function for this. This may involve actually naming and name-scoping all our parameters...


Discussion:
Worth checking out:

`gpflow.utilities.multiple_assign(input: tf.Module, vars_dict: Dict[str, tf.Tensor])`

Usage:
```
# model_1 copied to model_2

param_dict = gpflow.utilities.leaf_components(model_1)
gpflow.utilities.multiple_assign(model_2, param_dict)
```

I think the misleading thing is the function name `leaf_components`, which should be called something like: `get_params_dict` or smth.
It can be done with,

```python
params = gpflow.utilities.parameter_dict(m1)
gpflow.utilities.multiple_assign(m2, params)
```

I hope it resolves the question, closing it. Please feel free to re-open the issue, if this doesn't help.

Resolving patch:
diff --git a/doc/source/notebooks/intro_to_gpflow2.pct.py b/doc/source/notebooks/intro_to_gpflow2.pct.py
--- a/doc/source/notebooks/intro_to_gpflow2.pct.py
+++ b/doc/source/notebooks/intro_to_gpflow2.pct.py
@@ -72,10 +72,10 @@ def enumerated_logdir(_logdir_id: int = [0]):
 # %% [markdown]
 # ## Loading data using TensorFlow Datasets
 #
-# For this example, we create a synthetic dataset (noisy sine function): 
+# For this example, we create a synthetic dataset (noisy sine function):
 
 # %%
-def noisy_sin(x): 
+def noisy_sin(x):
     return tf.math.sin(x) + 0.1 * tf.random.normal(x.shape, dtype=default_float())
 
 num_train_data, num_test_data = 100, 500
@@ -117,7 +117,7 @@ def noisy_sin(x):
 # %% [markdown]
 # ## Define a GP model
 #
-# In GPflow 2.0, we use `tf.Module` (or the very thin `gpflow.base.Module` wrapper) to build all our models, as well as their components (kernels, likelihoods, parameters, and so on). 
+# In GPflow 2.0, we use `tf.Module` (or the very thin `gpflow.base.Module` wrapper) to build all our models, as well as their components (kernels, likelihoods, parameters, and so on).
 
 # %%
 kernel = gpflow.kernels.SquaredExponential(variance=2.)
@@ -160,6 +160,12 @@ def noisy_sin(x):
 
 print_summary(model)  # same as print_summary(model, fmt="notebook")
 
+# %% [markdown]
+# Jupyter notebooks also format GPflow classes (that are subclasses of `gpflow.base.Module`) in the same nice way when at the end of a cell (this is independent of the `default_summary_fmt`):
+
+# %%
+model
+
 # %% [markdown]
 # ## Training using Gradient Tapes
 #
@@ -172,7 +178,7 @@ def noisy_sin(x):
     tape.watch(model.trainable_variables)
     obj = - model.elbo(data)
     grads = tape.gradient(obj, model.trainable_variables)
-    
+
 optimizer.apply_gradients(zip(grads, model.trainable_variables))
 
 
@@ -189,7 +195,7 @@ def optimization_step(model: gpflow.models.SVGP, batch: Tuple[tf.Tensor, tf.Tens
 
 
 # %% [markdown]
-# We can use the functionality of TensorFlow Datasets to define a simple training loop that iterates over batches of the training dataset: 
+# We can use the functionality of TensorFlow Datasets to define a simple training loop that iterates over batches of the training dataset:
 
 # %%
 def simple_training_loop(model: gpflow.models.SVGP, epochs: int = 1, logging_epoch_freq: int = 10):
@@ -208,7 +214,7 @@ def simple_training_loop(model: gpflow.models.SVGP, epochs: int = 1, logging_epo
 simple_training_loop(model, epochs=10, logging_epoch_freq=2)
 
 # %% [markdown]
-# ## Monitoring 
+# ## Monitoring
 #
 # We can monitor the training procedure using `tf.summary`. First we create a summary writer object through which we can write scalars and images.
 
@@ -217,7 +223,7 @@ def simple_training_loop(model: gpflow.models.SVGP, epochs: int = 1, logging_epo
 
 samples_input = tf.cast(np.linspace(0, 10, 100).reshape(100, 1), default_float())
 
-def monitored_training_loop(model: gpflow.models.SVGP, logdir: str, 
+def monitored_training_loop(model: gpflow.models.SVGP, logdir: str,
                             epochs: int = 1, logging_epoch_freq: int = 10,
                             num_samples: int = 10):
     summary_writer = tf.summary.create_file_writer(logdir)
@@ -236,7 +242,7 @@ def monitored_training_loop(model: gpflow.models.SVGP, logdir: str,
                 mean, var = model.predict_f(samples_input)
                 samples = model.predict_f_samples(samples_input, num_samples)
                 fig = plotting_regression(X, Y, samples_input, mean, var, samples)
-                
+
                 summary_matplotlib_image(dict(model_samples=fig), step=epoch)
                 tf.summary.scalar('elbo', data=model.elbo(data), step=epoch)
                 tf.summary.scalar('likelihood/variance', data=model.likelihood.variance, step=epoch)
@@ -304,7 +310,7 @@ def checkpointing_training_loop(model: gpflow.models.SVGP,
                                 step_var: Optional[tf.Variable] = None):
     tf_optimization_step = tf.function(optimization_step)
     batches = iter(train_dataset)
-    
+
     for epoch in range(epochs):
         for step in range(num_batches_per_epoch):
             tf_optimization_step(model, next(batches))
@@ -312,7 +318,7 @@ def checkpointing_training_loop(model: gpflow.models.SVGP,
                 step_var.assign(epoch * num_batches_per_epoch + step + 1)
         if epoch_var is not None:
             epoch_var.assign(epoch + 1)
-            
+
         epoch_id = epoch + 1
         if epoch_id % logging_epoch_freq == 0:
             ckpt_path = manager.save()
@@ -337,4 +343,22 @@ def checkpointing_training_loop(model: gpflow.models.SVGP,
     ckpt.restore(recorded_checkpoint)
     print(f"{i} restored model from epoch {int(epoch_var)} [step:{int(step_var)}] : ELBO training set {model.elbo(data)}")
 
+# %% [markdown]
+# ## Copying (hyper)parameter values between models
+#
+# It is easy to interact with the set of all parameters of a model or a subcomponent programmatically.
+#
+# The following returns a dictionary of all parameters within
+
+# %%
+model = gpflow.models.SGPR(data, kernel=kernel, inducing_variable=inducing_variable)
+
+# %%
+gpflow.utilities.parameter_dict(model)
+
+# %% [markdown]
+# Such a dictionary can be assigned back to this model (or another model with the same tree of parameters) as follows:
+
 # %%
+params = gpflow.utilities.parameter_dict(model)
+gpflow.utilities.multiple_assign(model, params)
diff --git a/doc/source/notebooks/theory/upper_bound.pct.py b/doc/source/notebooks/theory/upper_bound.pct.py
--- a/doc/source/notebooks/theory/upper_bound.pct.py
+++ b/doc/source/notebooks/theory/upper_bound.pct.py
@@ -34,6 +34,8 @@
 import logging
 logging.disable(logging.WARNING)
 
+np.random.seed(1)
+
 from FITCvsVFE import getTrainingTestData
 
 # %%
@@ -80,7 +82,7 @@ def plot_model(m, name=""):
     objective = tf.function(autograph=False)(lambda: - vfe.log_marginal_likelihood())
     gpflow.optimizers.Scipy().minimize(objective, vfe.trainable_variables,
                                        options=dict(disp=False, maxiter=ci_niter(1000)))
-    
+
     vfe_lml.append(vfe.log_likelihood().numpy())
     vupper_lml.append(vfe.upper_bound().numpy())
     vfe_hyps.append([(p.name, p.numpy()) for p in vfe.trainable_parameters])
@@ -107,32 +109,27 @@ def plot_model(m, name=""):
 fvfe_lml = []  # Fixed vfe lml
 fvupper_lml = []  # Fixed upper lml
 
-init_params = [p.numpy() for p in vfe.parameters]  # TODO: provide convenience function
+init_params = gpflow.utilities.parameter_dict(vfe)
+
+# cannot copy this due to shape mismatch with different numbers of inducing points between models:
+del init_params['.inducing_variable.Z']
 
 for M in fMs:
     Zinit = vfe.inducing_variable.Z.numpy()[:M, :]
     Zinit = np.vstack((Zinit, X[np.random.permutation(len(X))[:(M - len(Zinit))], :].copy()))
-    
+
     vfe = gpflow.models.SGPR((X, Y), gpflow.kernels.SquaredExponential(), inducing_variable=Zinit)
-    
-    # copy hyperparameters from optimized model:
-    
-    for p, val in zip(vfe.parameters, init_params):
-        if p is vfe.inducing_variable.Z:
-            # would fail due to shape mismatch with different numbers of inducing points between models
-            continue
-        p.assign(val)
-
-    # keep our own augmented set of inducing point locations:
-    vfe.inducing_variable.Z = gpflow.Parameter(Zinit, dtype=gpflow.default_float())
-    
+
+    # copy hyperparameters (omitting inducing_variable.Z) from optimized model:
+    gpflow.utilities.multiple_assign(vfe, init_params)
+
     set_trainable(vfe.kernel, False)
     set_trainable(vfe.likelihood, False)
-    
+
     objective = tf.function(autograph=False)(lambda: - vfe.log_marginal_likelihood())
     gpflow.optimizers.Scipy().minimize(objective, vfe.trainable_variables,
                                        options=dict(disp=False, maxiter=ci_niter(1000)))
-    
+
     fvfe_lml.append(vfe.log_likelihood().numpy())
     fvupper_lml.append(vfe.upper_bound().numpy())
     print("%i" % M, end=" ")
@@ -159,7 +156,9 @@ def plot_model(m, name=""):
 vfe = gpflow.models.SGPR((X, Y), gpflow.kernels.SquaredExponential(), inducing_variable=single_inducing_point)
 objective = tf.function(autograph=False)(lambda: - vfe.log_marginal_likelihood())
 gpflow.optimizers.Scipy().minimize(objective, vfe.trainable_variables,
-                                   options=dict(maxiter=ci_niter(1000)))
+                                   options=dict(maxiter=ci_niter(1000)), jit=False)
+# Note that we need to set jit=False here due to a discrepancy in tf.function jitting
+# see https://github.com/GPflow/GPflow/issues/1260
 
 print("Lower bound: %f" % vfe.log_likelihood().numpy())
 print("Upper bound: %f" % vfe.upper_bound().numpy())
diff --git a/gpflow/base.py b/gpflow/base.py
--- a/gpflow/base.py
+++ b/gpflow/base.py
@@ -34,9 +34,8 @@ def trainable_parameters(self):
         return tuple(self._flatten(predicate=_IS_TRAINABLE_PARAMETER))
 
     def _repr_html_(self):
-        from IPython.display import HTML
         from .utilities import tabulate_module_summary
-        return HTML(tabulate_module_summary(self, tablefmt='html'))
+        return tabulate_module_summary(self, tablefmt='html')
 
     def _repr_pretty_(self, p, cycle):
         from .utilities import tabulate_module_summary


Test patch: