and

$$
\begin{align*}
& \partial_{\theta_l} \partial_{\mu}^i [h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(v) \\
&= \sum_{j=1}^{d} \partial_{\theta_l} \partial_{x_j} h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)}) \partial_{\mu}^i (\tilde{X}_s^{t,x,\mu}(\theta))^j \\
&\quad + \sum_{j,k=1}^{d} \partial_{x_j,x_k}^2 h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)}) \partial_{\mu}^i (\tilde{X}_s^{t,x,\mu}(\theta))^j \partial_{\theta_l} (\tilde{X}_s^{t,x,\mu}(\theta))^k \\
&\quad + \sum_{j,k=1}^{d} \hat{\mathbb{E}} [ [\partial_{\mu} \partial_{x_j} h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})]_k (\tilde{X}_s^{t,\xi}(\theta)) \partial_{\theta_l} (\tilde{X}_s^{t,\xi}(\theta))^k ] \partial_{\mu}^i (\tilde{X}_s^{t,x,\mu}(\theta))^j \\
&\quad + \sum_{j=1}^{d} \partial_{x_j} h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)}) \partial_{\theta_l} \partial_{\mu}^i (\tilde{X}_s^{t,x,\mu}(\theta))^j \\
&\quad + \sum_{j=1}^{d} \hat{\mathbb{E}} [ \partial_{\theta_l} \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,v,\mu}(\theta)) \partial_{v_i} (\tilde{X}_s^{t,v,\mu}(\theta))^j ] \\
&\quad + \sum_{j,k=1}^{d} \hat{\mathbb{E}} [ \partial_{x_k} \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,v,\mu}(\theta)) \partial_{v_i} (\tilde{X}_s^{t,v,\mu}(\theta))^j ] \partial_{\theta_l} (\tilde{X}_s^{t,x,\mu})^k \\
&\quad + \sum_{j,k=1}^{d} \hat{\mathbb{E}} \hat{\mathbb{E}} [ \partial_{\mu}^k \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,v,\mu}(\theta), \tilde{X}_s^{t,\xi}(\theta)) \partial_{v_i} (\tilde{X}_s^{t,v,\mu}(\theta))^j \partial_{\theta_l} (\tilde{X}_s^{t,\xi}(\theta))^k ] \\
&\quad + \sum_{j,k=1}^{d} \hat{\mathbb{E}} [ \partial_{v_k} \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,v,\mu}(\theta)) \partial_{v_i} (\tilde{X}_s^{t,v,\mu}(\theta))^j \partial_{\theta_l} (\tilde{X}_s^{t,v,\mu})^k ] \\
&\quad + \sum_{j=1}^{d} \hat{\mathbb{E}} [ \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,v,\mu}(\theta)) \partial_{\theta_l} \partial_{v_i} (\tilde{X}_s^{t,v,\mu}(\theta))^j ] \\
&\quad + \sum_{j=1}^{d} \int_{\mathbb{R}^d} \hat{\mathbb{E}} [ \partial_{\theta_l} \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,x',\mu}(\theta)) \partial_{\mu}^i (\tilde{X}_s^{t,x',\mu}(\theta))^j (v) ] \mu(dx') \\
&\quad + \sum_{j,k=1}^{d} \int_{\mathbb{R}^d} \hat{\mathbb{E}} [ \partial_{x_k} \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,x',\mu}(\theta)) \partial_{\theta_l} (\tilde{X}_s^{t,x,\mu}(\theta))^k \partial_{\mu}^i (\tilde{X}_s^{t,x',\mu}(\theta))^j (v) ] \mu(dx') \\
&\quad + \sum_{j,k=1}^{d} \int_{\mathbb{R}^d} \hat{\mathbb{E}} \hat{\mathbb{E}} [ \partial_{\mu}^k \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,x',\mu}(\theta), \tilde{X}_s^{t,\xi}(\theta)) \partial_{\mu}^i (\tilde{X}_s^{t,x',\mu}(\theta))^j (v) \partial_{\theta_l} (\tilde{X}_s^{t,\xi}(\theta))^k ] \mu(dx') \\
&\quad + \sum_{j,k=1}^{d} \int_{\mathbb{R}^d} \hat{\mathbb{E}} [ \partial_{v_k} \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,x',\mu}(\theta)) \partial_{\theta_l} (\tilde{X}_s^{t,x',\mu}(\theta))^k \partial_{\mu}^i (\tilde{X}_s^{t,x',\mu}(\theta))^j (v) ] \mu(dx') \\
&\quad + \sum_{j=1}^{d} \int_{\mathbb{R}^d} \hat{\mathbb{E}} [ \partial_{\mu}^j h_\theta(s, \tilde{X}_s^{t,x,\mu}(\theta), \mathbb{P}_{\tilde{X}_s^{t,\xi}(\theta)})(\tilde{X}_s^{t,x',\mu}(\theta)) \partial_{\theta_l} \partial_{\mu}^i (\tilde{X}_s^{t,x',\mu}(\theta))^j (v) ] \mu(dx').
\end{align*}
$$

In the above identity, $\tilde{X}_s^{t,\xi}(\theta)$ stands for a random variable independent of $(\tilde{X}_s^{t,x,\mu}, \tilde{X}_s^{t,v,\mu}, \tilde{X}_s^{t,x',\mu})$
$\partial_{\theta} \partial_{x}^{2} V_{\theta}(t, x, \mu)$, $\partial_{x}^{2} \partial_{\theta} V_{\theta}(t, x, \mu)$, $\partial_{\theta} \partial_{\mu} V_{\theta}(t, x, \mu)(v)$, $\partial_{\mu} \partial_{\theta} V_{\theta}(t, x, \mu)(v)$, $\partial_{\theta} \partial_{v} \partial_{\mu} V_{\theta}(t, x, \mu)(v)$, $\partial_{v} \partial_{\mu} \partial_{\theta} V_{\theta}(t, x, \mu)(v)$ exist and are locally Lipschitz continuous. Hence, from Clairaut's theorem, we deduce that $\partial_{\theta} \partial_{x} V_{\theta}(t, x, \mu) = \partial_{x} \partial_{\theta} V_{\theta}(t, x, \mu)$, $\partial_{\theta} \partial_{x}^{2} V_{\theta}(t, x, \mu) = \partial_{x}^{2} \partial_{\theta} V_{\theta}(t, x, \mu)$, $\partial_{\theta} \partial_{\mu} V_{\theta}(t, x, \mu)(v) = \partial_{\mu} \partial_{\theta} V_{\theta}(t, x, \mu)(v)$ and $\partial_{\theta} \partial_{v} \partial_{\mu} V_{\theta}(t, x, \mu)(v) = \partial_{v} \partial_{\mu} \partial_{\theta} V_{\theta}(t, x, \mu)(v)$ for all $t, x, \mu, \theta, v$.

Moreover, from Assumption 3.1 and Lemma A.1, there exist $q$ and $C$ such that for any $t, x, \mu, v$ and any $\theta \in \mathcal{K}$, $\mathcal{K}$ being a compact subset of $\Theta$

$$|\partial_{\theta} V_{\theta}(t, x, \mu)| \leq C(1 + |x|^2 + M_2(\mu)^q), \qquad (A.8)$$

$$|\partial_{\theta} \partial_{x} V_{\theta}(t, x, \mu)| + |\partial_{\theta} \partial_{\mu} V_{\theta}(t, x, \mu)(v)| \leq C(1 + |x| + |v| + M_2(\mu)^q), \qquad (A.9)$$

and

$$|\partial_{\theta} \partial_{x}^{2} V_{\theta}(t, x, \mu)| + |\partial_{\theta} \partial_{v} \partial_{\mu} V_{\theta}(t, x, \mu)(v)| \leq C(1 + |v| + M_2(\mu)^q). \qquad (A.10)$$

Now, differentiating with respect to $\theta$ both sides of (2.6), we deduce that $\theta \mapsto \partial_{t} V_{\theta}(t, x, \mu)$ is differentiable with a derivative $\partial_{\theta} \partial_{t} V_{\theta}(t, x, \mu)$ being continuous with respect to $t, x, \mu, \theta$. Also, taking $\pi = \pi_{\theta}$ and differentiating with respect to $\theta$ both sides of the identity of (A.7) (using Lemma A.1 together with the estimates (A.8), (A.9), (A.10) and the dominated convergence theorem to differentiate the right-hand side therein) and then passing to the limit as $h \downarrow 0$, we get that $t \mapsto \partial_{\theta} V_{\theta}(t, x, \mu)$ is differentiable with a derivative $\partial_{t} \partial_{\theta} V_{\theta}(t, x, \mu)$ being continuous with respect to $t, x, \mu, \theta$. We thus conclude that the two derivatives $\partial_{\theta} \partial_{t} V_{\theta}(t, x, \mu)$ and $\partial_{t} \partial_{\theta} V_{\theta}(t, x, \mu)$ coincide for all $t, x, \mu, \theta$.

### A.3 Proof of Theorem 3.1

Step 1: We start from the PDE characterisation of $V_{\theta}$ in Proposition 2.1 that we write as

$$\int_A \{ \mathcal{L}_{\theta}^a [V_{\theta}](t, x, \mu) + f(x, \mu, a) + \lambda \log p_{\theta}(t, x, \mu, a) \} \pi_{\theta}(da|t, x, \mu) = 0, \qquad (A.11)$$

where

$$\mathcal{L}_{\theta}^{a}[\varphi](t, x, \mu) = -\beta\varphi(t, x, \mu) + \partial_{t}\varphi(t, x, \mu) + b(x, \mu, a) \cdot D_{x}\varphi(t, x, \mu) + \frac{1}{2}\sigma\sigma^{\intercal}(x, \mu, a) : D_{x}^{2}\varphi(t, x, \mu) \\ + \mathbb{E}_{\xi \sim \mu} \left[ b_{\theta}(t, \xi, \mu) \cdot \partial_{\mu}\varphi(t, x, \mu)(\xi) + \frac{1}{2}\Sigma_{\theta}(t, \xi, \mu) : \partial_{v}\partial_{\mu}\varphi(t, x, \mu)(\xi) \right],$$

recalling that $b_{\theta}(t, x, \mu) = \int_{A} b(x, \mu, a) \pi_{\theta}(da|t, x, \mu)$, $\Sigma_{\theta}(t, x, \mu) = \int_{A} (\sigma\sigma^{\intercal})(x, \mu, a) \pi_{\theta}(da|t, x, \mu)$.

For any fixed $t, x, \mu$, we now differentiate w.r.t. $\theta \in \Theta$ both sides of (A.11) to get a new system of linear PDEs satisfied by $G_{\theta}$. In particular, using the identity

$$\nabla_{\theta} [ \mathcal{L}_{\theta}^{a}[V_{\theta}](t, x, \mu) ] = \mathcal{L}_{\theta}^{a}[G_{\theta}](t, x, \mu) + \mathcal{H}_{\theta}[V_{\theta}](t, x, \mu),$$

together with (3.1) and the dominated convergence theorem, we get

$$\int_A \left\{ \mathcal{L}_{\theta}^a[G_{\theta}](t, x, \mu) + \mathcal{H}_{\theta}[V_{\theta}](t, x, \mu) + [\mathcal{L}_{\theta}^a[V_{\theta}](t, x, \mu) + f(x, \mu, a) + \lambda \log p_{\theta}(t, x, \mu, a)] \nabla_{\theta} \log p_{\theta}(t, x, \mu, a) \right\} \pi_{\theta}(da|t, x, \mu) = 0, \qquad (A.12)$$
with the same law as $\tilde{X}_s^{t, \xi}$.

Then, starting from the expression of $V_\theta$ in (A.3) (with $\pi = \pi_\theta$), the dominated convergence theorem guarantees that the derivatives $(t, \theta, x, \mu, v) \mapsto \partial_\theta \partial_x V_\theta(t, x, \mu)$, $\partial_x \partial_\theta V_\theta(t, x, \mu)$, $\partial_\theta \partial_x^2 V_\theta(t, x, \mu)$, $\partial_x^2 \partial_\theta V_\theta(t, x, \mu)$, $\partial_\theta \partial_\mu V_\theta(t, x, \mu)(v)$, $\partial_\mu \partial_\theta V_\theta(t, x, \mu)(v)$, $\partial_\theta \partial_v \partial_\mu V_\theta(t, x, \mu)(v)$, $\partial_v \partial_\mu \partial_\theta V_\theta(t, x, \mu)(v)$ exist and are locally Lipschitz continuous. Hence, from Clairaut's theorem, we deduce that $\partial_\theta \partial_x V_\theta(t, x, \mu) = \partial_x \partial_\theta V_\theta(t, x, \mu)$, $\partial_\theta \partial_x^2 V_\theta(t, x, \mu) = \partial_x^2 \partial_\theta V_\theta(t, x, \mu)$, $\partial_\theta \partial_\mu V_\theta(t, x, \mu)(v) = \partial_\mu \partial_\theta V_\theta(t, x, \mu)(v)$ and $\partial_\theta \partial_v \partial_\mu V_\theta(t, x, \mu)(v) = \partial_v \partial_\mu \partial_\theta V_\theta(t, x, \mu)(v)$ for all $t, x, \mu, \theta, v$.

Moreover, from Assumption 3.1 and Lemma A.1, there exist $q$ and $C$ such that for any $t, x, \mu, v$ and any $\theta \in K$, $K$ being a compact subset of $\Theta$

$$|\partial_{\theta} V_{\theta}(t, x, \mu)| \leq C(1 + |x|^2 + M_2(\mu)^q), \quad (A.8)$$

$$|\partial_{\theta} \partial_{x} V_{\theta}(t, x, \mu)| + |\partial_{\theta} \partial_{\mu} V_{\theta}(t, x, \mu)(v)| \leq C(1 + |x| + |v| + M_2(\mu)^q), \quad (A.9)$$

and

$$|\partial_{\theta} \partial_{x}^{2} V_{\theta}(t, x, \mu)| + |\partial_{\theta} \partial_{v} \partial_{\mu} V_{\theta}(t, x, \mu)(v)| \leq C(1 + |v| + M_{2}(\mu)^{q}). \quad (A.10)$$

Now, differentiating with respect to $\theta$ both sides of (2.6), we deduce that $\theta \mapsto \partial_t V_\theta(t, x, \mu)$ is differentiable with a derivative $\partial_\theta \partial_t V_\theta(t, x, \mu)$ being continuous with respect to $t, x, \mu, \theta$. Also, taking $\pi = \pi_\theta$ and differentiating with respect to $\theta$ both sides of the identity of (A.7) (using Lemma A.1 together with the estimates (A.8), (A.9), (A.10) and the dominated convergence theorem to differentiate the right-hand side therein) and then passing to the limit as $h \downarrow 0$, we get that $t \mapsto \partial_\theta V_\theta(t, x, \mu)$ is differentiable with a derivative $\partial_t \partial_\theta V_\theta(t, x, \mu)$ being continuous with respect to $t, x, \mu, \theta$. We thus conclude that the two derivatives $\partial_\theta \partial_t V_\theta(t, x, \mu)$ and $\partial_t \partial_\theta V_\theta(t, x, \mu)$ coincide for all $t, x, \mu, \theta$.

## A.3 Proof of Theorem 3.1

*Step 1:* We start from the PDE characterisation of $V_\theta$ in Proposition 2.1 that we write as

$$\int_A \{\mathcal{L}_\theta^a[V_\theta](t, x, \mu) + f(x, \mu, a) + \lambda \log p_\theta(t, x, \mu, a)\} \pi_\theta(\mathrm{d}a|t, x, \mu) = 0, \quad (A.11)$$

where

$$\mathcal{L}_{\theta}^{a}[\varphi](t, x, \mu) = -\beta\varphi(t, x, \mu) + \partial_t\varphi(t, x, \mu) + b(x, \mu, a) \cdot D_x\varphi(t, x, \mu) + \frac{1}{2}\sigma\sigma^{\intercal}(x, \mu, a) : D_x^2\varphi(t, x, \mu) \\ + \mathbb{E}_{\xi \sim \mu} \left[ b_{\theta}(t, \xi, \mu) \cdot \partial_{\mu}\varphi(t, x, \mu)(\xi) + \frac{1}{2}\Sigma_{\theta}(t, \xi, \mu) : \partial_{v}\partial_{\mu}\varphi(t, x, \mu)(\xi) \right],$$

recalling that $b_\theta(t, x, \mu) = \int_A b(x, \mu, a) \pi_\theta(\mathrm{d}a|t, x, \mu)$, $\Sigma_\theta(t, x, \mu) = \int_A (\sigma\sigma^{\intercal})(x, \mu, a) \pi_\theta(\mathrm{d}a|t, x, \mu)$.

For any fixed $t, x, \mu$, we now differentiate w.r.t. $\theta \in \Theta$ both sides of (A.11) to get a new system of linear PDEs satisfied by $G_\theta$. In particular, using the identity

$$\nabla_{\theta} \left[ \mathcal{L}_{\theta}^{a}[V_{\theta}](t, x, \mu) \right] = \mathcal{L}_{\theta}^{a}[G_{\theta}](t, x, \mu) + \mathcal{H}_{\theta}[V_{\theta}](t, x, \mu),$$
with terminal condition $G_θ(T, x, \mu) = 0$. Note that we have used the fact that

$$\int_A \nabla_\theta \log p_\theta(t, x, \mu, a) \pi_\theta(\mathrm{d}a|t, x, \mu) = \nabla_\theta \int_A \pi_\theta(\mathrm{d}a|t, x, \mu) = 0,$$

and the above PDE is a system of $D$ equations, where $\mathcal{L}_\theta^a[G_\theta]$ denotes the operator applied to each component of the $\mathbb{R}^D$-valued function $G_\theta$.

Step 2: Denote by

$$\tilde{F}_\theta(t, x, \mu, a) = \{\mathcal{L}_\theta^a[V_\theta](t, x, \mu) + f(x, \mu, a) + \lambda \log p_\theta(t, x, \mu, a)\} \nabla_\theta \log p_\theta(t, x, \mu, a) + \mathcal{H}_\theta[V_\theta](t, x, \mu),$$

and

$$\tilde{f}_{\pi_{\theta}}(t, x, \mu) = \int_{A} \tilde{F}_{\theta}(t, x, \mu, a) \pi_{\theta}(\mathrm{d}a|t, x, \mu),$$

so that the linear PDE (A.12) satisfied by $G_\theta$ now writes

$$\mathcal{L}_{\pi_{\theta}}[G_{\theta}](t, x, \mu) + \tilde{f}_{\pi_{\theta}}(t, x, \mu) = 0,$$

with terminal condition $G_\theta(T, x, \mu) = 0$. Observe that the above PDE is similar to (2.6). In order to obtain the announced probabilistic representation formula, we first apply the chain rule formula on the strip $[t, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d)$, see e.g. Proposition 5.102 in Carmona and Delarue (2018a), to $(e^{-\beta s}G_\theta(s, \tilde{X}_s^{t,x,\mu}, \mathbb{P}_{\tilde{X}_s^{t,\xi}}))_{s \in [t,T]}$ using the estimates (A.8) and (A.10). We thus obtain

$$d(e^{-\beta s} G_\theta(s, \tilde{X}_s^{t,x,\mu}, \mathbb{P}_{\tilde{X}_s^{t,\xi}})) = -e^{-\beta s} \tilde{f}_{\pi_\theta}(s, \tilde{X}_s^{t,x,\mu}, \mathbb{P}_{\tilde{X}_s^{t,\xi}}) \mathrm{d}s \\ + e^{-\beta s} \partial_x G_\theta(s, \tilde{X}_s^{t,x,\mu}, \mathbb{P}_{\tilde{X}_s^{t,\xi}})^\top \sigma_{\pi_\theta}(\tilde{X}_s^{t,x,\mu}, \mathbb{P}_{\tilde{X}_s^{t,\xi}}) \mathrm{d}W_s.$$

Observe that (A.9) together with the fact that for any $\theta \in \mathbb{R}^D$, $|\sigma_{\pi_\theta}(x, \mu)| \le C(1 + |x| + M_2(\mu))$, for some constant $C$, directly yields that the stochastic integral is a square integrable martingale. Hence, integrating from $t$ to $T$ both sides of the above and using the facts that $G_\theta(T, x, \mu) = 0$ and $\mathbb{P}_{\tilde{X}_s^{t,\xi}} = \mathbb{P}_{X_s^{t,\xi}}$, $\mathbb{P}_{\tilde{X}_s^{t,x,\mu}} = \mathbb{P}_{X_s^{t,x,\mu}}$, we eventually deduce

$$G_{\theta}(t, x, \mu) = \mathbb{E}_{\alpha \sim \pi_{\theta}} \left[ \int_{t}^{T} e^{-\beta(s-t)} \tilde{F}_{\theta}(s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}, \alpha_{s}) \mathrm{d}s \right].$$

Step 3: On the other hand, applying again the chain rule formula to $V_\theta(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}})$, when $\alpha \sim \pi_\theta$, see e.g. Proposition 5.102 in Carmona and Delarue (2018a), we have

$$dV_{\theta}(s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}) = (\mathcal{L}_{\theta}^{\alpha_{s}}[V_{\theta}](s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}) + \beta V_{\theta}(s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}})) \mathrm{d}s \\ + D_{x}V_{\theta}(s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}})^{\top}\sigma(X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}, \alpha_{s}) \mathrm{d}W_{s}, \quad t \le s \le T,$$
together with (3.1) and the dominated convergence theorem, we get

$$ \int_A \left\{ \mathcal{L}_\theta^a[G_\theta](t, x, \mu) + \mathcal{H}_\theta[V_\theta](t, x, \mu) + [\mathcal{L}_\theta^a[V_\theta](t, x, \mu) + f(x, \mu, a) + \lambda \log p_\theta(t, x, \mu, a)] \nabla_\theta \log p_\theta(t, x, \mu, a) \right\} \pi_\theta(\mathrm{d}a|t, x, \mu) = 0, \quad (A.12) $$

with terminal condition $G_\theta(T, x, \mu) = 0$. Note that we have used the fact that

$$ \int_A \nabla_\theta \log p_\theta(t, x, \mu, a) \pi_\theta(\mathrm{d}a|t, x, \mu) = \nabla_\theta \int_A \pi_\theta(\mathrm{d}a|t, x, \mu) = 0, $$

and the above PDE is a system of $D$ equations, where $\mathcal{L}_\theta^a[G_\theta]$ denotes the operator applied to each component of the $\mathbb{R}^D$-valued function $G_\theta$.

Step 2: Denote by

$$ \tilde{F}_\theta(t, x, \mu, a) = \{ \mathcal{L}_\theta^a[V_\theta](t, x, \mu) + f(x, \mu, a) + \lambda \log p_\theta(t, x, \mu, a) \} \nabla_\theta \log p_\theta(t, x, \mu, a) + \mathcal{H}_\theta[V_\theta](t, x, \mu), $$

and

$$ \tilde{f}_{\pi_{\theta}}(t, x, \mu) = \int_{A} \tilde{F}_{\theta}(t, x, \mu, a) \pi_{\theta}(\mathrm{d}a|t, x, \mu), $$

so that the linear PDE (A.12) satisfied by $G_\theta$ now writes

$$ \mathcal{L}_{\pi_{\theta}}[G_{\theta}](t, x, \mu) + \tilde{f}_{\pi_{\theta}}(t, x, \mu) = 0, $$

with terminal condition $G_\theta(T, x, \mu) = 0$. Observe that the above PDE is similar to (2.6). In order to obtain the announced probabilistic representation formula, we first apply the chain rule formula on the strip $[t, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d)$, see e.g. Proposition 5.102 in [4], to $(e^{-\beta s}G_\theta(s, \tilde{X}_s^{t,x,\mu}, \mathbb{P}_{\tilde{X}_s^{t,\xi}}))_{s \in [t,T]}$ using the estimates (A.8) and (A.10). We thus obtain

$$ d(e^{-\beta s}G_{\theta}(s, \tilde{X}_{s}^{t,x,\mu}, \mathbb{P}_{\tilde{X}_{s}^{t,\xi}})) = -e^{-\beta s} \tilde{f}_{\pi_{\theta}}(s, \tilde{X}_{s}^{t,x,\mu}, \mathbb{P}_{\tilde{X}_{s}^{t,\xi}}) \mathrm{d}s \\ + e^{-\beta s} \partial_{x}G_{\theta}(s, \tilde{X}_{s}^{t,x,\mu}, \mathbb{P}_{\tilde{X}_{s}^{t,\xi}})^{\mathrm{T}} \sigma_{\pi_{\theta}}(\tilde{X}_{s}^{t,x,\mu}, \mathbb{P}_{\tilde{X}_{s}^{t,\xi}}) \mathrm{d}W_{s}. $$

Observe that (A.9) together with the fact that for any $\theta \in \mathbb{R}^D$, $|\sigma_{\pi_\theta}(x, \mu)| \le C(1 + |x| + M_2(\mu))$, for some constant $C$, directly yields that the stochastic integral is a square integrable martingale. Hence, integrating from $t$ to $T$ both sides of the above and using the facts that $G_\theta(T, x, \mu) = 0$ and $\mathbb{P}_{\tilde{X}_s^{t,\xi}} = \mathbb{P}_{X_s^{t,\xi}}$, $\mathbb{P}_{\tilde{X}_s^{t,x,\mu}} = \mathbb{P}_{X_s^{t,x,\mu}}$, we eventually deduce

$$ G_{\theta}(t, x, \mu) = \mathbb{E}_{\alpha \sim \pi_{\theta}} \left[ \int_{t}^{T} e^{-\beta(s-t)} \tilde{F}_{\theta}(s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}, \alpha_{s}}) \mathrm{d}s \right]. $$
and thus by definition of $\tilde{F}_{\theta}$

$$ 
\begin{aligned}
& \int_t^T e^{-\beta(s-t)} \tilde{F}_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s) \mathrm{d}s \\
&= \int_t^T e^{-\beta(s-t)} \nabla_{\theta} \log(p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s)) \left( \mathrm{d}V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) - \beta V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) \right. \\
& \qquad \left. + \mathcal{H}_{\theta}[V_{\theta}](s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) \right) \mathrm{d}s \\
&\quad + \int_t^T e^{-\beta(s-t)} \nabla_{\theta} \log(p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s)) \left( f(X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s) + \lambda \log(p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s)) \right) \mathrm{d}s \\
&\quad - \int_t^T e^{-\beta(s-t)} \nabla_{\theta} \log(p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s)) D_x V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}})^{\top} \sigma(X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s) \mathrm{d}W_s.
\end{aligned}
$$

Note that (3.2) as well as the bound $|D_x V_{\theta}(s, x, \mu)| \le C(1 + |x| + |\mu|^q)$, for some $q \ge 0$, directly deduced from the identity (A.4) and Assumption 2.1, guarantees that the stochastic integral appearing in the right-hand side of the above identity is a square integrable martingale. Hence, taking expectation in both sides of the above identity eventually yields

$$ 
\begin{aligned}
G_{\theta}(t, x, \mu) &:= \mathbb{E}_{\alpha \sim \pi_{\theta}} \left[ \int_t^T e^{-\beta(s-t)} \nabla_{\theta} \log p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s) \left\{ \mathrm{d}V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) \right. \right. \\
& \qquad \left. + \left[ f(X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s) + \lambda \log p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}\alpha_s) - \beta V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) \right] \mathrm{d}s \right] \\
& \qquad + \int_t^T e^{-\beta(s-t)} \mathcal{H}_{\theta}[V_{\theta}](s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) \mathrm{d}s.
\end{aligned}
$$

This proves the announced probabilistic representation formula for $G_{\theta}$.

## Appendix B. Linear quadratic mean-field control with randomised controls and entropy regularisation

A stochastic policy is a probability transition kernel from $[0, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d)$ into $A = \mathbb{R}^m$, i.e., a measurable function $\pi: (t, x, \mu) \in [0, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d) \mapsto \pi(\cdot|t, x, \mu) \in \mathcal{P}(\mathbb{R}^m)$. We denote by $\Pi$ the set of stochastic policies $\pi$ with densities $p$ with respect to the Lebesgue measure on $\mathbb{R}^m$: $\pi(\mathrm{d}a|t, x, \mu) = p(t, x, \mu, a)\mathrm{d}a$. We say that the process $\alpha = (\alpha_t)_t$ is a randomised feedback control generated from a stochastic policy $\pi \in \Pi$, denoted by $\alpha \sim \pi$, if at each time $t$, the action $\alpha_t$ is sampled (according to the $\sigma$-algebra $\mathcal{G}$) from the probability distribution $\pi(\cdot|t, X_t, \mathbb{P}_{X_t})$. The dynamics $X = X^\alpha$ follows a linear mean-field dynamics with coefficients $b(x, \mu, a) = \bar{b}(x, \bar{\mu}, a)$, $\sigma(x, \mu, a) = \bar{\sigma}(x, \bar{\mu}, a)$ in the form

$$ \bar{b}(x, \bar{x}, a) = Bx + \bar{B}\bar{x} + Ca, \quad \bar{\sigma}(x, \bar{x}, a) = \gamma + Dx + \bar{D}\bar{x} + Fa, $$

for $(x, \mu, \bar{x}, a) \in \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d) \times \mathbb{R}^d \times \mathbb{R}^m$, where we denote by $\bar{\mu} = \int x\mu(\mathrm{d}x)$, $B, \bar{B}, D, \bar{D}$ are constant matrices in $\mathbb{R}^{d \times d}$, $C, F$ are constant matrices in $\mathbb{R}^{d \times m}$, $\gamma$ is a constant in $\mathbb{R}^d$.

Given a stochastic policy $\pi \in \Pi$, we consider the functional cost $V^\pi$ with entropy regulariser defined in (2.5) with quadratic functions $f(x, \mu, a) = \bar{f}(x, \bar{\mu}, a)$ and $g(x, \mu) =$
Step 3: On the other hand, applying again the chain rule formula to $V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}})$, when $\alpha \sim \pi_{\theta}$, see e.g. Proposition 5.102 in [4], we have

$$dV_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) = (\mathcal{L}_{\theta}^{\alpha_s}[V_{\theta}](s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) + \beta V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}})) ds \\ + D_x V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}})^{\intercal} \sigma(X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}, \alpha_s) dW_s, \quad t \le s \le T,$$

and thus by definition of $\tilde{F}_{\theta}$

$$\begin{aligned} & \int_t^T e^{-\beta(s-t)} \tilde{F}_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}, \alpha_s) ds \\ &= \int_t^T e^{-\beta(s-t)} \nabla_{\theta} \log(p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}, \alpha_s)) (dV_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) - \beta V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}) \\ & \qquad + \mathcal{H}_{\theta}[V_{\theta}](s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}})) ds \\ & \quad + \int_t^T e^{-\beta(s-t)} \nabla_{\theta} \log(p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}, \alpha_s)) (f(X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}, \alpha_s) + \lambda \log(p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}, \alpha_s))) ds \\ & \quad - \int_t^T e^{-\beta(s-t)} \nabla_{\theta} \log(p_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}, \alpha_s)) D_x V_{\theta}(s, X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}})^{\intercal} \sigma(X_s^{t,x,\mu}, \mathbb{P}_{X_s^{t,\xi}}, \alpha_s) dW_s. \end{aligned}$$

Note that (3.2) as well as the bound $|D_x V_{\theta}(s, x, \mu)| \le C(1 + |x| + |\mu|^q)$, for some $q \ge 0$, directly deduced from the identity (A.4) and Assumption 2.1, guarantees that the stochastic integral appearing in the right-hand side of the above identity is a square integrable martingale. Hence, taking expectation in both sides of the above identity eventually yields

$$\begin{aligned} G_{\theta}(t, x, \mu) := \mathbb{E}_{\alpha \sim \pi_{\theta}} \left[ \int_{t}^{T} e^{-\beta(s-t)} \nabla_{\theta} \log p_{\theta}(s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}, \alpha_{s}) \left\{ dV_{\theta}(s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}) \right. \right. \\ \left. + \left[ f(X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}, \alpha_{s}) + \lambda \log p_{\theta}(s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}, \alpha_{s}) - \beta V_{\theta}(s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}) \right] ds \right\} \\ \left. + \int_{t}^{T} e^{-\beta(s-t)} \mathcal{H}_{\theta}[V_{\theta}](s, X_{s}^{t,x,\mu}, \mathbb{P}_{X_{s}^{t,\xi}}) ds \right]. \end{aligned}$$

This proves the announced probabilistic representation formula for $G_{\theta}$.

## B Linear quadratic mean-field control with randomised controls and entropy regularisation

A stochastic policy is a probability transition kernel from $[0, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d)$ into $A = \mathbb{R}^m$, i.e., a measurable function $\pi : (t, x, \mu) \in [0, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d) \mapsto \pi(\cdot|t, x, \mu) \in \mathcal{P}(\mathbb{R}^m)$. We denote by $\Pi$ the set of stochastic policies $\pi$ with densities $p$ with respect to the Lebesgue measure on $\mathbb{R}^m$: $\pi(\mathrm{d}a|t, x, \mu) = p(t, x, \mu, a)\mathrm{d}a$. We say that the process $\alpha = (\alpha_t)_t$ is a randomised
$\bar{g}(x, \bar{\mu})$:

$$
\begin{align*}
\bar{f}(x, \bar{x}, a) &= x^\top Q x + \bar{x}^\top \bar{Q} \bar{x} + a^\top N a + 2a^\top I x + 2a^\top \bar{I} \bar{x} + 2M.x + 2H.a, \\
\bar{g}(x, \bar{x}) &= x^\top P x + \bar{x}^\top \bar{P} \bar{x} + 2L.x,
\end{align*}
$$

where $N$ is a symmetric matrix in $S_+^m$, $I, \bar{I} \in \mathbb{R}^{m \times d}$, $Q, \bar{Q}, P, \bar{P}$ are symmetric matrices in $S^d, M, L \in \mathbb{R}^d, H \in \mathbb{R}^m$, assumed to satisfy the conditions:

(H1) (i) There exists $\delta > 0$ s.t.

$$
N \geq \delta I_m, \quad P \geq 0, \quad Q - I^\top N^{-1} I \geq 0.
$$

or (ii) $n = m = 1, I = 0, F \neq 0, Q \geq 0, P > 0$.

(H2) (i) There exists $\delta > 0$ s.t.

$$
N \geq \delta I_m, \quad P + \bar{P} \geq 0, \quad (Q + \bar{Q}) - (I + \bar{I})^\top N^{-1} (I + \bar{I}) \geq 0.
$$

or (ii) $I + \bar{I} = 0, F \neq 0, Q + \bar{Q} \geq 0, P + \bar{P} \geq 0, P > 0$.

The solution to the LQ mean-field control problem with entropy regulariser is then given
by the following theorem:

**Theorem B.1** Let Assumptions **(H1)**-(**H2**) hold. Then, the value function is equal to

$$
v(t, x, \mu) := \inf_{\pi \in \Pi} V^{\pi}(t, x, \mu) = (x - \bar{\mu})^{\top} K(t) (x - \bar{\mu}) + \bar{\mu}^{\top} \Lambda(t) \bar{\mu} + 2Y(t)^{\top} x + R(t),
$$

for $(t, x, \mu) \in [0, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d)$, where the quadruple $(K, \Lambda, Y, R)$ valued in $(S_+^d, S_+^d, \mathbb{R}^d, \mathbb{R})$
is solution on $[0, T]$ to the system of Riccati equations:

$$
\left\{
\begin{array}{l}
\dot{K}(t) - \beta K(t) + Q + K(t)B + B^\top K(t) + D^\top K(t)D \\
\quad -(I + C^\top K(t) + F^\top K(t)D)^\top (N + F^\top K(t)F)^{-1} (I + C^\top K(t) + F^\top K(t)D) = 0, \\
\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t) \hat{B} + \hat{B}^\top \Lambda(t) + \hat{D}^\top K(t) \hat{D} \\
\quad -(\hat{I} + C^\top \Lambda(t) + F^\top K(t) \hat{D})^\top (N + F^\top K(t) F)^{-1} (\hat{I} + C^\top \Lambda(t) + F^\top K(t) \hat{D}) = 0 \quad (\text{B.1}) \\
\dot{Y}(t) - \beta Y(t) + M + \hat{B}^\top Y(t) + \hat{D}^\top K(t) \gamma \\
\quad -(\hat{I} + C^\top \Lambda(t) + F^\top K(t) \hat{D})^\top (N + F^\top K(t) F)^{-1} (H + C^\top Y(t) + F^\top K(t) \gamma) = 0 \\
\dot{R}(t) - \beta R(t) + \gamma^\top K(t) \gamma + \frac{\lambda m}{2} \log(2\pi) - \frac{\lambda}{2} \log \left| \frac{\lambda}{2 \det(N + F^\top K(t)F)} \right| \\
\quad -(H + C^\top Y(t) + F^\top K(t) \gamma)^\top (N + F^\top K(t) F)^{-1} (H + C^\top Y(t) + F^\top K(t) \gamma) = 0
\end{array}
\right.
$$

with the terminal condition $(K(T), \Lambda(T), Y(T), R(T)) = (P, \hat{P}, L, 0)$, where we set $\hat{I} := I + \bar{I}$, $\hat{B} := B + \bar{B}$, $\hat{D} := D + \bar{D}$, $\hat{Q} := Q + \bar{Q}$, $\hat{P} := P + \bar{P}$.

Moreover, the optimal stochastic policy follows a Gaussian distribution:

$$
\pi^*(\cdot | t, x, \mu) = \mathcal{N} \left( -S(t)^{-1} (U(t)x + (\hat{U}(t) - U(t))\bar{\mu} + O(t)); \frac{\lambda}{2} S(t)^{-1} \right), \quad (B.2)
$$

where we set

$$
S(t) := N + F^{\top} K(t) F, \quad O(t) := H + C^{\top} Y(t) + F^{\top} K(t) \gamma
$$

$$
U(t) := I + C^{\top} K(t) + F^{\top} K(t) D, \quad \hat{U}(t) := \hat{I} + C^{\top} \Lambda(t) + F^{\top} K(t) \hat{D}.
$$
feedback control generated from a stochastic policy $\pi \in \Pi$, denoted by $\alpha \sim \pi$, if at each time $t$, the action $\alpha_t$ is sampled (according to the $\sigma$-algebra $\mathcal{G}$) from the probability distribution $\pi(\cdot|t, X_t, \mathbb{P}_{X_t})$. The dynamics $X = X^\alpha$ follows a linear mean-field dynamics with coefficients $b(x, \mu, a) = \bar{b}(x, \bar{\mu}, a)$, $\sigma(x, \mu, a) = \bar{\sigma}(x, \bar{\mu}, a)$ in the form

$$ \bar{b}(x, \bar{x}, a) = Bx + \bar{B}\bar{x} + Ca, \quad \bar{\sigma}(x, \bar{x}, a) = \gamma + Dx + \bar{D}\bar{x} + Fa, $$

for $(x, \mu, \bar{x}, a) \in \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d) \times \mathbb{R}^d \times \mathbb{R}^m$, where we denote by $\bar{\mu} = \int x\mu(dx)$, $B, \bar{B}, D, \bar{D}$ are constant matrices in $\mathbb{R}^{d \times d}$, $C, F$ are constant matrices in $\mathbb{R}^{d \times m}$, $\gamma$ is a constant in $\mathbb{R}^d$.

Given a stochastic policy $\pi \in \Pi$, we consider the functional cost $V^\pi$ with entropy regulariser defined in (2.5) with quadratic functions $f(x, \mu, a) = \bar{f}(x, \bar{\mu}, a)$ and $g(x, \mu) = \bar{g}(x, \bar{\mu})$:

$$ \bar{f}(x, \bar{x}, a) = x^\top Q x + \bar{x}^\top \bar{Q} \bar{x} + a^\top N a + 2a^\top I x + 2a^\top \bar{I} \bar{x} + 2M.x + 2H.a, \\ \bar{g}(x, \bar{x}) = x^\top P x + \bar{x}^\top \bar{P} \bar{x} + 2L.x, $$

where $N$ is a symmetric matrix in $S_+^m$, $I, \bar{I} \in \mathbb{R}^{m \times d}$, $Q, \bar{Q}, P, \bar{P}$ are symmetric matrices in $S^d$, $M, L \in \mathbb{R}^d$, $H \in \mathbb{R}^m$, assumed to satisfy the conditions:

(H1) (i) There exists $\delta > 0$ s.t.

$$ N \geq \delta I_m, \quad P \geq 0, \quad Q - I^\top N^{-1} I \geq 0. $$

or (ii) $n = m = 1$, $I = 0$, $F \neq 0$, $Q \geq 0$, $P > 0$.

(H2) (i) There exists $\delta > 0$ s.t.

$$ N \geq \delta I_m, \quad P + \bar{P} \geq 0, \quad (Q + \bar{Q}) - (I + \bar{I})^\top N^{-1} (I + \bar{I}) \geq 0. $$

or (ii) $I + \bar{I} = 0$, $F \neq 0$, $Q + \bar{Q} \geq 0$, $P + \bar{P} \geq 0$, $P > 0$.

The solution to the LQ mean-field control problem with entropy regulariser is then given by the following theorem:

**Theorem B.1** Let Assumptions (H1)-(H2) hold. Then, the value function is equal to

$$ v(t, x, \mu) := \inf_{\pi \in \Pi} V^{\pi}(t, x, \mu) = (x - \bar{\mu})^{\top} K(t)(x - \bar{\mu}) + \bar{\mu}^{\top} \Lambda(t) \bar{\mu} + 2Y(t)^{\top} x + R(t), $$

for $(t, x, \mu) \in [0, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d)$, where the quadruple $(K, \Lambda, Y, R)$ valued in $(S_+^d, S_+^d, \mathbb{R}^d, \mathbb{R})$
**Remark B.1** Conditions (H1) and (H2) ensure the existence and uniqueness of a solution $(K, \Lambda)$ to the matrix Riccati equation in (B.1) satisfying $K \ge 0$, $\Lambda \ge 0$ (hence $S(t)^{-1}$ is well-defined). Given $(K, \Lambda)$, the equations for $(Y, R)$ are simply linear ODEs.

**Proof of Theorem B.1.** We adapt the arguments in Basei and Pham (2019) to our case with randomised controls and entropy regulariser.

Step 1. Let us consider the function defined on $[0, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d)$ by $w(t, x, \mu) = \bar{w}(t, x, \bar{\mu})$, where $\bar{w}$ is defined on $[0, T] \times \mathbb{R}^d \times \mathbb{R}^d$ by

$$ \bar{w}(t, x, \bar{x}) = (x - \bar{x})^{\top} K(t)(x - \bar{x}) + \bar{x}^{\top} \Lambda(t) \bar{x} + 2Y(t)^{\top} x + R(t), $$

for some functions (to be determined later) $K$, $\Lambda$, $Y$ and $R$ on $[0, T]$, and valued on $\mathbb{S}_+^d$, $\mathbb{S}_+^d$, $\mathbb{R}^d$, and $\mathbb{R}$. Fix $(t_0, x_0, \mu_0) \in [0, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d)$, and $\xi_0 \in L^2(\mathcal{F}_{t_0}; \mathbb{R}^d) \sim \mu_0$. Given $\pi \in \Pi$ with density $p$, and a randomised control $\alpha \sim \pi$, we consider the process

$$ S_t^\alpha := e^{-\beta(t-t_0)} \bar{w}(t, X_t^{t_0, x_0, \mu_0}, \bar{X}_t^{t_0, \mu_0}) + \int_{t_0}^t e^{-\beta(s-t_0)} [\bar{f}(X_s^{t_0, x_0, \mu_0}, \bar{X}_s^{t_0, \mu_0}, \alpha_s) \\ + \lambda \int_{\mathbb{R}^m} (\log p_t(a)) p_t(a) da] ds, $$

for $t_0 \le t \le T$, where we set $p_t(a) = p(t, X_t^{t_0, x_0, \mu_0}, \mathbb{P}_{X_t^{t_0, \xi_0}, a})$, and $\bar{X}_t^{t_0, \mu_0} := \mathbb{E}_{\alpha \sim \pi}[X_t^{t_0, \xi_0}]$ which follows the dynamics:

$$ d\bar{X}_t = (\hat{B}\bar{X}_t + C\bar{\alpha}_t)dt, $$

with $\bar{\alpha}_t := \mathbb{E}_{\alpha \sim \pi}[\alpha_t]$.

Step 2. We apply Itô's formula to $S_t^\alpha$ for $\alpha \sim \pi$, and take the expectation to get

$$ d\mathbb{E}_{\alpha \sim \pi}[S_t^\alpha] = e^{-\beta(t-t_0)} \mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^\alpha] dt, \qquad (B.3) $$

with

$$ \mathcal{D}_t^\alpha = -\beta \bar{w}(t, X_t, \bar{X}_t) + \frac{d}{dt} \mathbb{E}_{\alpha \sim \pi}[\bar{w}(t, X_t, \bar{X}_t)] + \bar{f}(X_t, \bar{X}_t, \alpha_t) + \lambda \int_{\mathbb{R}^m} (\log p_t(a)) p_t(a) da, $$

where we omit the dependence on $t_0, x_0, \mu_0$ of $X$ and $\bar{X}$ to alleviate notations. By applying Itô's formula to $\bar{w}(t, X_t, \bar{X}_t)$, recalling the quadratic forms of $\bar{w}, \bar{f}$, and using the linear dynamics of $X$ and $\bar{X}$, we obtain similarly as in Basei and Pham (2019) (after careful but
is solution on $[0, T]$ to the system of Riccati equations:

$$
\left\{
\begin{aligned}
\dot{K}(t) - \beta K(t) + Q + K(t)B + B^\top K(t) + D^\top K(t)D \\
& - (I + C^\top K(t) + F^\top K(t)D)^\top (N + F^\top K(t)F)^{-1} (I + C^\top K(t) + F^\top K(t)D) = 0, \\
\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t) \hat{B} + \hat{B}^\top \Lambda(t) + \hat{D}^\top K(t) \hat{D} \\
& - (\hat{I} + C^\top \Lambda(t) + F^\top K(t) \hat{D})^\top (N + F^\top K(t) F)^{-1} (\hat{I} + C^\top \Lambda(t) + F^\top K(t) \hat{D}) = 0 \\
\dot{Y}(t) - \beta Y(t) + M + \hat{B}^\top Y(t) + \hat{D}^\top K(t) \gamma \\
& - (\hat{I} + C^\top \Lambda(t) + F^\top K(t) \hat{D})^\top (N + F^\top K(t) F)^{-1} (H + C^\top Y(t) + F^\top K(t) \gamma) = 0 \\
\dot{R}(t) - \beta R(t) + \gamma^\top K(t) \gamma + \frac{\lambda m}{2} \log(2\pi) - \frac{\lambda}{2} \log \left| \frac{\lambda}{2 \det(N + F^\top K(t)F)} \right| \\
& - (H + C^\top Y(t) + F^\top K(t) \gamma)^\top (N + F^\top K(t) F)^{-1} (H + C^\top Y(t) + F^\top K(t) \gamma) = 0
\end{aligned}
\right.
\quad (B.1)
$$

with the terminal condition $(K(T), \Lambda(T), Y(T), R(T)) = (P, \hat{P}, L, 0)$, where we set $\hat{I} := I + \bar{I}$,
$\hat{B} := B + \bar{B}$, $\hat{D} := D + \bar{D}$, $\hat{Q} := Q + \bar{Q}$, $\hat{P} := P + \bar{P}$.

Moreover, the optimal stochastic policy follows a Gaussian distribution:

$$
\pi^*(\cdot|t, x, \mu) = \mathcal{N}\left(-S(t)^{-1}\left(U(t)x + (\hat{U}(t) - U(t))\bar{\mu} + O(t)\right); \frac{\lambda}{2}S(t)^{-1}\right), \quad (B.2)
$$

where we set

$$
\begin{align*}
S(t) &:= N + F^{\top} K(t) F, & O(t) &:= H + C^{\top} Y(t) + F^{\top} K(t) \gamma \\
U(t) &:= I + C^{\top} K(t) + F^{\top} K(t) D, & \hat{U}(t) &:= \hat{I} + C^{\top} \Lambda(t) + F^{\top} K(t) \hat{D}.
\end{align*}
$$

**Remark B.1** *Conditions (H1) and (H2) ensure the existence and uniqueness of a solution (K, Λ) to the matrix Riccati equation in (B.1) satisfying K ≥ 0, Λ ≥ 0 (hence S(t)⁻¹ is well-defined). Given (K, Λ), the equations for (Y, R) are simply linear ODEs.*

**Proof of Theorem B.1.** We adapt the arguments in [2] to our case with randomised controls and entropy regulariser.

Step 1. Let us consider the function defined on $[0, T] \times \mathbb{R}^d \times \mathcal{P}_2(\mathbb{R}^d)$ by $w(t, x, \mu) = \bar{w}(t, x, \bar{\mu})$, where $\bar{w}$ is defined on $[0, T] \times \mathbb{R}^d \times \mathbb{R}^d$ by

$$
\bar{w}(t, x, \bar{x}) = (x - \bar{x})^{\top} K(t) (x - \bar{x}) + \bar{x}^{\top} \Lambda(t) \bar{x} + 2Y(t)^{\top} x + R(t),
$$

for some functions (to be determined later) K, Λ, Y and R on [0, T], and valued on S<sub>+</sub><sup>d</sup>, S<sub>+</sub><sup>d</sup>,
R<sup>d</sup>, and R. Fix (t<sub>0</sub>, x<sub>0</sub>, μ<sub>0</sub>) ∈ [0, T] × R<sup>d</sup> × P<sub>2</sub>(R<sup>d</sup>), and ξ<sub>0</sub> ∈ L<sup>2</sup>(F<sub>t_0</sub>; R<sup>d</sup>) ~ μ<sub>0</sub>. Given π ∈ Π
with density p, and a randomised control α ~ π, we consider the process

$$
S_t^\alpha := e^{-\beta(t-t_0)} \bar{w}(t, X_t^{t_0, x_0, \mu_0}, \bar{X}_t^{t_0, \mu_0}) + \int_{t_0}^t e^{-\beta(s-t_0)} [\bar{f}(X_s^{t_0, x_0, \mu_0}, \bar{X}_s^{t_0, \mu_0}, \alpha_s) \\
+ \lambda \int_{\mathbb{R}^m} (\log p_t(a)) p_t(a) da] ds,
$$
straightforward computations):

$$
\begin{align}
\mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^\alpha] &= \mathbb{E}_{\alpha \sim \pi} \left[ (X_t - \bar{X}_t)^\intercal (\dot{K}(t) - \beta K(t) + Q + K(t)B + B^\intercal K(t) + D^\intercal K(t)D)(X_t - \bar{X}_t) \right. \nonumber \\
&\qquad + \bar{X}_t^\intercal (\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t) \hat{B} + \hat{B}^\intercal \Lambda(t) + \hat{D}^\intercal K(t) \hat{D}) \bar{X}_t \nonumber \\
&\qquad + 2(\dot{Y}(t) - \beta Y(t) + M + \hat{B}^\intercal Y(t) + \hat{D}^\intercal K(t) \gamma)^\intercal X_t + \dot{R}(t) - \beta R(t) + \gamma^\intercal K(t) \gamma \nonumber \\
&\qquad + \alpha_t^\intercal S(t) \alpha_t + 2 \alpha_t^\intercal (U(t)(X_t - \bar{X}_t) + \hat{U}(t) \bar{X}_t + O(t)) + \lambda \int_{\mathbb{R}^m} (\log p_t(a)) p_t(a) da \nonumber \\
&= \mathbb{E}_{\alpha \sim \pi} \left[ (X_t - \bar{X}_t)^\intercal (\dot{K}(t) - \beta K(t) + Q + K(t)B + B^\intercal K(t) + D^\intercal K(t)D)(X_t - \bar{X}_t) \right. \nonumber \\
&\qquad + \bar{X}_t^\intercal (\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t) \hat{B} + \hat{B}^\intercal \Lambda(t) + \hat{D}^\intercal K(t) \hat{D}) \bar{X}_t \nonumber \\
&\qquad + 2(\dot{Y}(t) - \beta Y(t) + M + \hat{B}^\intercal Y(t) + \hat{D}^\intercal K(t) \gamma)^\intercal X_t + \dot{R}(t) - \beta R(t) + \gamma^\intercal K(t) \gamma \nonumber \\
&\qquad + \int_{\mathbb{R}^m} [\phi_t(a) + \lambda \log p_t(a)] p_t(a) da \tag{B.4}
\end{align}
$$

where we used in the last equality the fact that $\alpha \sim \pi$, and set $\phi_t(a) := a^\intercal S(t)a + 2a^\intercal \chi_t$ with $\chi_t := U(t)(X_t - \bar{X}_t) + \hat{U}(t)\bar{X}_t + O(t)$.

Step 3. Let $\phi$ be a quadratic function on $\mathbb{R}^m$: $\phi(a) = a^\intercal S a + 2a^\intercal \chi$ for some positive-definite matrix $S \in \mathbb{S}_+^m$, and $\chi \in \mathbb{R}^m$, and denote by $\mathcal{D}_2(\mathbb{R}^m)$ the set of square integrable density functions on $\mathbb{R}^m$, i.e., the set of nonnegative measurable functions $p$ on $\mathbb{R}^m$ s.t. $\int_{\mathbb{R}^m} p(a) da = 1$, and $\int_{\mathbb{R}^m} |a|^2 p(a) da < \infty$. Let us consider the cost functional on $\mathcal{D}_2(\mathbb{R}^m)$ defined by

$$C_{\phi}(p) := \int_{\mathbb{R}^m} [\phi(a) + \lambda \log p(a)] p(a) da.$$

Then, the minimizer of $C_\phi$ is achieved with $p^* \in \mathcal{D}_2(\mathbb{R}^m)$ given by

$$p^*(a) = \frac{\exp\left(-\frac{1}{\lambda}\phi(a)\right)}{\int_{\mathbb{R}^m} \exp\left(-\frac{1}{\lambda}\phi(a)\right) da}, \quad a \in \mathbb{R}^m,$$

which corresponds to the density of a Gaussian distribution

$$\pi^* = \mathcal{N}\left(-S^{-1}\chi; \frac{\lambda}{2}S^{-1}\right). \qquad (B.5)$$

We refer to Lemma 13 in Jia and Zhou (2023) for this derivation. The infimum of $C_\phi$ is then equal to

$$\inf_{p \in \mathcal{D}_2(\mathbb{R}^m)} C_\phi(p) = C_\phi(p^*) = -\chi^\intercal S^{-1} \chi - \frac{\lambda m}{2} \log(2\pi) - \frac{\lambda}{2} \log \left| \frac{\lambda}{2 \det(S)} \right|. \quad (B.6)$$

Step 4. Notice that under (H1), the matrix $S(t) = N + F^\intercal K(t)F$ is positive-definite for $K \ge 0$, and $p_t(\cdot) \in \mathcal{D}_2(\mathbb{R}^m)$ a.s. for $t \in [t_0, T]$. From (B.4) and (B.6), we then have for all $\pi$
for $t_0 \le t \le T$, where we set $p_t(a) = p(t, X_t^{t_0, x_0, \mu_0}, \mathbb{P}_{X_t^{t_0, \xi_0}, a})$, and $\bar{X}_t^{t_0, \mu_0} := \mathbb{E}_{\alpha \sim \pi}[X_t^{t_0, \xi_0}]$ which follows the dynamics:

$$d\bar{X}_t = (\hat{B}\bar{X}_t + C\bar{\alpha}_t)dt,$$

with $\bar{\alpha}_t := \mathbb{E}_{\alpha \sim \pi}[\alpha_t]$.

Step 2. We apply Itô's formula to $S_t^\alpha$ for $\alpha \sim \pi$, and take the expectation to get

$$d\mathbb{E}_{\alpha \sim \pi}[S_t^\alpha] = e^{-\beta(t-t_0)}\mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^\alpha]dt, \quad (B.3)$$

with

$$\mathcal{D}_t^\alpha = -\beta \bar{w}(t, X_t, \bar{X}_t) + \frac{d}{dt} \mathbb{E}_{\alpha \sim \pi}[\bar{w}(t, X_t, \bar{X}_t)] + \bar{f}(X_t, \bar{X}_t, \alpha_t) + \lambda \int_{\mathbb{R}^m} (\log p_t(a)) p_t(a) da,$$

where we omit the dependence on $t_0, x_0, \mu_0$ of $X$ and $\bar{X}$ to alleviate notations. By applying Itô's formula to $\bar{w}(t, X_t, \bar{X}_t)$, recalling the quadratic forms of $\bar{w}, \bar{f}$, and using the linear dynamics of $X$ and $\bar{X}$, we obtain similarly as in [2] (after careful but straightforward computations):

$$\begin{aligned} \mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^\alpha] &= \mathbb{E}_{\alpha \sim \pi} \left[ (X_t - \bar{X}_t)^\intercal (\dot{K}(t) - \beta K(t) + Q + K(t)B + B^\intercal K(t) + D^\intercal K(t)D)(X_t - \bar{X}_t) \right. \\ & \qquad \left. + \bar{X}_t^\intercal (\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t) \hat{B} + \hat{B}^\intercal \Lambda(t) + \hat{D}^\intercal K(t) \hat{D}) \bar{X}_t \right. \\ & \qquad \left. + 2(\dot{Y}(t) - \beta Y(t) + M + \hat{B}^\intercal Y(t) + \hat{D}^\intercal K(t) \gamma)^\intercal X_t + \dot{R}(t) - \beta R(t) + \gamma^\intercal K(t) \gamma \right. \\ & \qquad \left. + \alpha_t^\intercal S(t) \alpha_t + 2\alpha_t^\intercal (U(t)(X_t - \bar{X}_t) + \hat{U}(t)\bar{X}_t + O(t)) + \lambda \int_{\mathbb{R}^m} (\log p_t(a)) p_t(a) da \right] \\ &= \mathbb{E}_{\alpha \sim \pi} \left[ (X_t - \bar{X}_t)^\intercal (\dot{K}(t) - \beta K(t) + Q + K(t)B + B^\intercal K(t) + D^\intercal K(t)D)(X_t - \bar{X}_t) \right. \\ & \qquad \left. + \bar{X}_t^\intercal (\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t) \hat{B} + \hat{B}^\intercal \Lambda(t) + \hat{D}^\intercal K(t) \hat{D}) \bar{X}_t \right. \\ & \qquad \left. + 2(\dot{Y}(t) - \beta Y(t) + M + \hat{B}^\intercal Y(t) + \hat{D}^\intercal K(t) \gamma)^\intercal X_t + \dot{R}(t) - \beta R(t) + \gamma^\intercal K(t) \gamma \right. \\ & \qquad \left. + \int_{\mathbb{R}^m} [\phi_t(a) + \lambda \log p_t(a)] p_t(a) da \right], \end{aligned} \quad (B.4)$$

where we used in the last equality the fact that $\alpha \sim \pi$, and set $\phi_t(a) := a^\intercal S(t)a + 2a^\intercal \chi_t$ with $\chi_t := U(t)(X_t - \bar{X}_t) + \hat{U}(t)\bar{X}_t + O(t)$.

Step 3. Let $\phi$ be a quadratic function on $\mathbb{R}^m$: $\phi(a) = a^\intercal Sa + 2a^\intercal \chi$ for some positive-definite matrix $S \in \mathbb{S}_+^m$, and $\chi \in \mathbb{R}^m$, and denote by $\mathcal{D}_2(\mathbb{R}^m)$ the set of square integrable density functions on $\mathbb{R}^m$, i.e., the set of nonnegative measurable functions $p$ on $\mathbb{R}^m$ s.t. $\int_{\mathbb{R}^m} p(a) da = 1$, and $\int_{\mathbb{R}^m} |a|^2 p(a) da < \infty$. Let us consider the cost functional on $\mathcal{D}_2(\mathbb{R}^m)$ defined by

$$C_{\phi}(p) := \int_{\mathbb{R}^m} [\phi(a) + \lambda \log p(a)] p(a) da.$$
$\in \Pi,$

$$
\begin{align*}
& \mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^{\alpha}] \\
&= \mathbb{E}_{\alpha \sim \pi} \left[ (X_t - \bar{X}_t)^{\top} (\dot{K}(t) - \beta K(t) + Q + K(t)B + B^{\top}K(t) + D^{\top}K(t)D)(X_t - \bar{X}_t) \right. \\
&\qquad \left. + \bar{X}_t^{\top} (\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t) \hat{B} + \hat{B}^{\top} \Lambda(t) + \hat{D}^{\top} K(t) \hat{D})(\bar{X}_t) \right. \\
&\qquad \left. + 2(\dot{Y}(t) - \beta Y(t) + M + \hat{B}^{\top} Y(t) + \hat{D}^{\top} K(t) \gamma)^{\top} X_t + \dot{R}(t) - \beta R(t) + \gamma^{\top} K(t) \gamma \right. \\
&\qquad \left. + C_{\phi_t}(p_t) \right] \\
&\geq \mathbb{E}_{\alpha \sim \pi} \left[ (X_t - \bar{X}_t)^{\top} (\dot{K}(t) - \beta K(t) + Q + K(t)B + B^{\top}K(t) + D^{\top}K(t)D - U(t)^{\top} S(t)^{-1} U(t))(X_t - \bar{X}_t) \right. \\
&\qquad \left. + \bar{X}_t^{\top} (\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t) \hat{B} + \hat{B}^{\top} \Lambda(t) + \hat{D}^{\top} K(t) \hat{D} - \hat{U}(t)^{\top} S(t)^{-1} \hat{U}(t))(\bar{X}_t) \right. \\
&\qquad \left. + 2(\dot{Y}(t) - \beta Y(t) + M + \hat{B}^{\top} Y(t) + \hat{D}^{\top} K(t) \gamma - O(t)^{\top} S(t)^{-1} \hat{U}(t))^{\top} X_t \right. \\
&\qquad \left. + \dot{R}(t) - \beta R(t) + \gamma^{\top} K(t) \gamma - O(t)^{\top} S(t)^{-1} O(t) - \frac{\lambda m}{2} \log(2\pi) - \frac{\lambda}{2} \log \left| \frac{\lambda}{2 \det(S(t))} \right| \right]. \tag{B.7}
\end{align*}
$$

Therefore, by taking $(K, \Lambda, Y, R)$ solution to (B.1), we see that the r.h.s. of (B.7) vanishes, which means that for all $\pi \in \Pi$, $\mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^\alpha] \ge 0$. Moreover, from (B.5), the equality in (B.7) holds true for the choice of $\pi^* \in \Pi$ as defined in (B.2), and thus

$$ \inf_{\pi \in \Pi} \mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^\alpha] = \mathbb{E}_{\alpha \sim \pi^*}[\mathcal{D}_t^\alpha] = 0, \quad t \in [t_0, T]. $$

From (B.3), this means that the function $t \mapsto \mathbb{E}_{\alpha \sim \pi}[S_t^\alpha]$ is nondecreasing on $[t_0, T]$ for any $\pi \in \Pi$, and constant on $[t_0, T]$ for $\pi = \pi^*$. By definition of $S^\alpha$, $V^\pi$, and noting that $\bar{w}(T, x, \bar{x}) = \bar{g}(x, \bar{x})$ from the terminal condition on $(K, \Lambda, Y, R)$, it follows that

$$ w(t_0, x_0, \mu_0) = \bar{w}(t_0, x_0, \bar{\mu}_0) = \mathbb{E}_{\alpha \sim \pi}[S_{t_0}^{\alpha}] \le \mathbb{E}_{\alpha \sim \pi}[S_T^{\alpha}] = V^{\pi}(t_0, x_0, \mu_0), \quad (B.8) $$

for any $\pi \in \Pi$, with equality in (B.8) for $\pi = \pi^*$. We conclude that

$$
\begin{align*}
\inf_{\pi \in \Pi} V^{\pi}(t_0, x_0, \mu_0) &= V^{\pi^*}(t_0, x_0, \mu_0) = w(t_0, x_0, \mu_0) \\
&= (x_0 - \bar{\mu}_0)^{\top} K(t_0) (x_0 - \bar{\mu}_0) + \bar{\mu}_0^{\top} \Lambda(t_0) \bar{\mu}_0 + 2Y(t_0)^{\top} x_0 + R(t_0).
\end{align*}
$$

□

## References

A. Angiuli, J.-P. Fouque, and M. Laurière. Unified reinforcement Q-learning for mean field game and control problems. *Mathematics of Control, Signals and Systems*, 34:217–271, 2022.

A. Angiuli, J.-P. Fouque, and M. Laurière. Reinforcement learning for mean field games, with applications to economics. In *Machine Learning and Data Sciences for Financial Markets: A Guide to Contemporary Practices*, page 393–425. Cambridge University Press, 2023.
Then, the minimizer of $C_{\phi}$ is achieved with $p^* \in \mathcal{D}_2(\mathbb{R}^m)$ given by

$$ p^*(a) = \frac{\exp\left(-\frac{1}{\lambda}\phi(a)\right)}{\int_{\mathbb{R}^m} \exp\left(-\frac{1}{\lambda}\phi(a)\right) da}, \quad a \in \mathbb{R}^m. \qquad (B.5) $$

Indeed, by considering the Lagrangian function associated to this minimization problem

$$ L_{\phi}(p, \nu) = C_{\phi}(p) - \nu \left( \int_{\mathbb{R}^m} p(a) da - 1 \right) = \int_{\mathbb{R}^m} \left[ \phi(a) + \lambda \log p(a) - \nu \right] p(a) da + \nu, $$

for $(p, \nu) \in \mathcal{D}_2(\mathbb{R}^m) \times \mathbb{R}$, we see that the minimization over $p$ is obtained pointwisely, i.e. inside the integral over $a \in \mathbb{R}^m$, hence leading to the first-order equations:

$$ \begin{cases} \phi(a) + \lambda \log p^*(a) - \nu^* + \lambda = 0, & a \in \mathbb{R}^m, \\ \int_{a \in \mathbb{R}^m} p^*(a) da = 1. \end{cases} $$

This yields the expression of $p^*$ in (B.5), which is actually the density of a Gaussian distribution

$$ \pi^* = \mathcal{N}\left(-S^{-1}\chi; \frac{\lambda}{2}S^{-1}\right). \qquad (B.6) $$

The infimum of $C_{\phi}$ is then equal to

$$ \inf_{p \in \mathcal{D}_2(\mathbb{R}^m)} C_{\phi}(p) = C_{\phi}(p^*) = -\chi^\top S^{-1} \chi - \frac{\lambda m}{2} \log(2\pi) - \frac{\lambda}{2} \log \left| \frac{\lambda}{2\det(S)} \right|. \qquad (B.7) $$

Step 4. Notice that under (H1), the matrix $S(t) = N + F^\top K(t)F$ is positive-definite for $K \ge 0$, and $p_t(\cdot) \in \mathcal{D}_2(\mathbb{R}^m)$ a.s. for $t \in [t_0, T]$. From (B.4) and (B.7), we then have for all $\pi \in \Pi$,

$$ \begin{aligned} & \mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^{\alpha}] \\ &= \mathbb{E}_{\alpha \sim \pi} \left[ (X_t - \bar{X}_t)^{\top} (\dot{K}(t) - \beta K(t) + Q + K(t)B + B^{\top}K(t) + D^{\top}K(t)D)(X_t - \bar{X}_t) \right. \\ & \qquad + \bar{X}_t^{\top} (\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t)\hat{B} + \hat{B}^{\top}\Lambda(t) + \hat{D}^{\top}K(t)\hat{D}) \bar{X}_t \\ & \qquad + 2(\dot{Y}(t) - \beta Y(t) + M + \hat{B}^{\top}Y(t) + \hat{D}^{\top}K(t)\gamma)^{\top} X_t + \dot{R}(t) - \beta R(t) + \gamma^{\top}K(t)\gamma \\ & \qquad + C_{\phi_t}(p_t) \right] \\ & \ge \mathbb{E}_{\alpha \sim \pi} \left[ (X_t - \bar{X}_t)^{\top} (\dot{K}(t) - \beta K(t) + Q + K(t)B + B^{\top}K(t) + D^{\top}K(t)D - U(t)^{\top}S(t)^{-1}U(t))(X_t - \bar{X}_t) \right. \\ & \qquad + \bar{X}_t^{\top} (\dot{\Lambda}(t) - \beta \Lambda(t) + \hat{Q} + \Lambda(t)\hat{B} + \hat{B}^{\top}\Lambda(t) + \hat{D}^{\top}K(t)\hat{D} - \hat{U}(t)^{\top}S(t)^{-1}\hat{U}(t))(X_t - \bar{X}_t) \\ & \qquad + 2(\dot{Y}(t) - \beta Y(t) + M + \hat{B}^{\top}Y(t) + \hat{D}^{\top}K(t)\gamma - O(t)^{\top}S(t)^{-1}\hat{U}(t))^{\top} X_t \\ & \qquad + \dot{R}(t) - \beta R(t) + \gamma^{\top}K(t)\gamma - O(t)^{\top}S(t)^{-1}O(t) - \frac{\lambda m}{2} \log(2\pi) - \frac{\lambda}{2} \log \left| \frac{\lambda}{2\det(S(t))} \right| \right]. \end{aligned} \qquad (B.8) $$
M. Basei and H. Pham. A Weak Martingale Approach to Linear-Quadratic McKean-Vlasov Stochastic Control Problems. *Journal of Optimization Theory and Applications*, 181(2): 347-382, 2019.

R. Carmona and F. Delarue. Forward Backward stochastic differential equations and controlled McKean-Vlasov dynamics. *Annals of Probability*, 43:2647-2700, 2015.

R. Carmona and F. Delarue. *Probabilistic Theory of Mean Field Games: vol. I, Mean Field FBSDEs, Control, and Games, Mean Field game with common noise and Master equations*. Springer, 2018a.

R. Carmona and F. Delarue. *Probabilistic Theory of Mean Field Games: vol. II, Mean Field game with common noise and Master equations*. Springer, 2018b.

R. Carmona and M. Laurière. Convergence analysis of machine learning algorithms for the numerical solution of mean field control and games: II—the finite horizon case. *The Annals of Applied Probability*, 32(6):4065-4105, 2022.

R. Carmona, J.-P. Fouque, and L. Sun. Mean field games and systemic risk. *Commun. Math. Sci.*, 13(4):911-933, 2015.

R. Carmona, M. Laurière, and Z. Tan. Linear-quadratic mean-field reinforcement learning: convergence of policy gradient methods. 2019.

R. Carmona, M. Laurière, and Z. Tan. Model-free mean-field reinforcement learning: mean-field MDP and mean-field Q-learning. *The Annals of Applied Probability*, 33(6B):5334-5381, 2023.

J.F. Chassagneux, D. Crisan, and F. Delarue. *A probabilistic approach to classical solutions of the master equation for large population equilibria*, volume 280. Memoirs of the AMS, 2022.

P.-E. Chaudru de Raynal and N. Frikha. From the backward Kolmogorov PDE on the Wasserstein space to propagation of chaos for McKean-Vlasov SDEs. *Journal de Mathématiques Pures et Appliquées*, 156:1-124, 2021.

P.-E. Chaudru de Raynal and N. Frikha. Well-posedness for some non-linear SDEs and related PDE on the Wasserstein space. *Journal de Mathématiques Pures et Appliquées*, 159:1-167, 2022.

D. Crisan and E. McMurray. Smoothing properties of McKean-Vlasov SDEs. *Probability Theory and Related Fields*, 171:97-148, 2018.

G. Dayanikli, M. Laurière, and J. Zhang. Deep learning for population-dependent controls in mean field control problems with common noise. In *Proceedings of the 23rd International Conference on Autonomous Agents and Multiagent Systems*, pages 2231-2233, 2024.

R. Elie, J. Pérolat, M. Laurière, M. Geist, and O. Pietquin. On the convergence of model free learning in mean field games. In *34th AAAI Conference on Artificial Intelligence, AAAI 2020*, 2020.
Therefore, by taking $(K, \Lambda, Y, R)$ solution to (B.1), we see that the r.h.s. of (B.8) vanishes, which means that for all $\pi \in \Pi$, $\mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^\alpha] \ge 0$. Moreover, from (B.6), the equality in (B.8) holds true for the choice of $\pi^* \in \Pi$ as defined in (B.2), and thus

$$ \inf_{\pi \in \Pi} \mathbb{E}_{\alpha \sim \pi}[\mathcal{D}_t^\alpha] = \mathbb{E}_{\alpha \sim \pi^*}[\mathcal{D}_t^\alpha] = 0, \quad t \in [t_0, T]. $$

From (B.3), this means that the function $t \mapsto \mathbb{E}_{\alpha \sim \pi}[S_t^\alpha]$ is nondecreasing on $[t_0, T]$ for any $\pi \in \Pi$, and constant on $[t_0, T]$ for $\pi = \pi^*$. By definition of $S^\alpha$, $V^\pi$, and noting that $\bar{w}(T, x, \bar{x}) = \bar{g}(x, \bar{x})$ from the terminal condition on $(K, \Lambda, Y, R)$, it follows that

$$ w(t_0, x_0, \mu_0) = \bar{w}(t_0, x_0, \bar{\mu}_0) = \mathbb{E}_{\alpha \sim \pi}[S_{t_0}^\alpha] \le \mathbb{E}_{\alpha \sim \pi}[S_T^\alpha] = V^\pi(t_0, x_0, \mu_0), \quad (B.9) $$

for any $\pi \in \Pi$, with equality in (B.9) for $\pi = \pi^*$. We conclude that

$$ \begin{aligned} \inf_{\pi \in \Pi} V^\pi(t_0, x_0, \mu_0) &= V^{\pi^*}(t_0, x_0, \mu_0) = w(t_0, x_0, \mu_0) \\ &= (x_0 - \bar{\mu}_0)^\top K(t_0)(x_0 - \bar{\mu}_0) + \bar{\mu}_0^\top \Lambda(t_0) \bar{\mu}_0 + 2Y(t_0)^\top x_0 + R(t_0). \end{aligned} $$

□

## References

[1] A. Angiuli, J.-P. Fouque, and M. Laurière. Unified reinforcement Q-learning for mean field game and control problems. *Mathematics of Control, Signals and Systems*, 34:217–271, 2022.

[2] M. Basei and H. Pham. A Weak Martingale Approach to Linear-Quadratic McKean-Vlasov Stochastic Control Problems. *Journal of Optimization Theory and Applications*, 181(2):347–382, 2019.

[3] R. Carmona and F. Delarue. Forward Backward stochastic differential equations and controlled McKean-Vlasov dynamics. *Annals of Probability*, 43:2647–2700, 2015.

[4] R. Carmona and F. Delarue. *Probabilistic Theory of Mean Field Games: vol. I, Mean Field FBSDEs, Control, and Games, Mean Field game with common noise and Master equations*. Springer, 2018.

[5] R. Carmona and F. Delarue. *Probabilistic Theory of Mean Field Games: vol. II, Mean Field game with common noise and Master equations*. Springer, 2018.

[6] R. Carmona, J.-P. Fouque, and L. Sun. Mean field games and systemic risk. *Commun. Math. Sci.*, 13(4):911–933, 2015.
N. Frikha, V. Konakov, and S. Menozzi. Well-posedness of some non-linear stable driven sdes. *Discrete and Continuous Dynamical Systems*, 41(2):849–898, 2021.

M. Germain, M. Laurière, H. Pham, and X. Warin. DeepSets and their derivative networks for solving symmetric PDEs. *Journal of Scientific Computing*, 91(63), 2022a.

M. Germain, J. Mikael, and X. Warin. Numerical resolution of McKean-Vlasov FBSDEs using neural networks. *Methodology and Computing in Applied Probability*, 2022b.

H. Gu, X. Guo, X. Wei, and R. Xu. Mean field controls with Q-learning for cooperative MARL: convergence and complexity analysis. *SIAM Journal on Mathematics of Data Science*, 3(4), 2021.

H. Gu, X. Guo, X. Wei, and R. Xu. Mean field multiagent reinforcement learning: a decentralized network approach. *Mathematics of Operations Research*, 2024.

X. Guo, R. Xu, and T. Zariphopoulou. Entropy regularization for mean field games with learning. *Mathematics of Operations research*, 47(4), 2022.

X. Guo, H. Pham, and X. Wei. Itô's formula for flows of measures on semimartingales. *Stochastic Processes and their Applications*, 159:350–390, 2023. doi: https://doi.org/10.1016/j.spa.2023.02.004.

J Han, R. Hu, and J. Long. Learning high-dimensional McKean-Vlasov forward-backward stochastic differential equations with general distribution dependence. *SIAM Journal on Numerical Analysis*, 62(1):1–24, 2024.

K. Hornik, M. Stinchcombe, and H. White. Multilayer feedforward networks are universal approximators. *Neural Networks*, 2:359–366, 1989.

Y. Jia and X. Y. Zhou. Policy gradient and actor critic learning in continuous time and space: theory and algorithms. *Journal of Machine learning Research*, 23(275):1–50, 2022a.

Y. Jia and X. Y. Zhou. Policy evaluation and temporal difference learning in continuous time and space: a martingale approach. *Journal of Machine learning Research*, 23(198): 1–55, 2022b.

Y. Jia and X. Y. Zhou. *q*-learning in continuous time. *Journal of Machine learning Research*, 24(161):1–61, 2023.

H. Pham and X. Warin. Mean-field neural networks-based algorithms for McKean-Vlasov control problems. *Journal of Machine learning*, 3:176–214, 2024a.

H. Pham and X. Warin. Actor-critic learning algorithms for mean-field control with moment neural networks. *Methodology and Computing in Applied Probability*, 27(13), 2024b.

H. Pham and X. Wei. Dynamic programming for optimal control of stochastic McKean-Vlasov dynamics. *SIAM Journal on Control and Optimization*, 55(2):1069–1101, 2017.

C. Reisinger, W. Stockinger, and Y. Zhang. A fest iterative PDE-based algorithm for feedback controls of nonsmooth mean-field control problems. *arXiv: 2108.06740*, 2021.
[7] R. Carmona and M. Laurière. Convergence analysis of machine learning algorithms for the numerical solution of mean-field control and games: II-the finite horizon case. *to appear in Annals of Applied Probability*, 2021.

[8] R. Carmona, M. Laurière, and Z. Tan. Model-free mean-field reinforcement learning: mean-field MDP and mean-field Q-learning. arXiv: 1910.12802v1, 2019.

[9] René Carmona, Mathieu Laurière, and Zongjun Tan. Linear-quadratic mean-field reinforcement learning: convergence of policy gradient methods. 2019.

[10] J.F. Chassagneux, D. Crisan, and F. Delarue. *A probabilistic approach to classical solutions of the master equation for large population equilibria*, volume 280. Memoirs of the AMS, 2022.

[11] P.-E. Chaudru de Raynal and N. Frikha. From the backward Kolmogorov PDE on the Wasserstein space to propagation of chaos for McKean-Vlasov SDEs. *Journal de Mathématiques Pures et Appliquées*, 156:1–124, 2021.

[12] P.-E. Chaudru de Raynal and N. Frikha. Well-posedness for some non-linear SDEs and related PDE on the Wasserstein space. *Journal de Mathématiques Pures et Appliquées*, 159:1–167, 2022.

[13] D. Crisan and E. McMurray. Smoothing properties of McKean–Vlasov SDEs. *Probability Theory and Related Fields*, 171:97–148, 2018.

[14] R. Elie, J. Perolat, M. Laurière, M. Geist, and O. Pietquin. On the convergence of model free learning in mean field games. Proceedings of AAAI, 2020.

[15] N. Frikha, V. Konakov, and S. Menozzi. Well-posedness of some non-linear stable driven sdes. *Discrete and Continuous Dynamical Systems*, 41(2):849–898, 2021.

[16] M. Germain, M. Laurière, H. Pham, and X. Warin. DeepSets and their derivative networks for solving symmetric PDEs. *Journal of Scientific Computing*, 91(63), 2022.

[17] M. Germain, J. Mikael, and X. Warin. Numerical resolution of McKean-Vlasov FBSDEs using neural networks. *Methodology and Computing in Applied Probability*, 2022.

[18] H. Gu, X. Guo, X. Wei, and R. Xu. Mean field controls with Q-learning for cooperative MARL: convergence and complexity analysis. *SIAM Journal on Mathematics of Data Science*, 3(4), 2021.

[19] X. Guo, H. Pham, and X. Wei. Itô's formula for flow of measures on semimartingales. *arXiv:2010.05288*, to appear in *Stochastic Processes and their Applications*, 2021.
Y. Sun. The exact law of large numbers via fubini extension and characterization of insurable risks. *Journal of Economic Theory*, 126(1):31–69, 2006.

R. Sutton and A. Barto. *Reinforcement Learning: An Introduction*. Cambridge, MA:MIT, 2018.

W. Tang and X Y. Zhou. Regret of exploratory and *q*-learning. *arXiv: 2411.01302*, 2024.

H. Wang, T. Zariphopoulou, and X. Y. Zhou. Reinforcement learning in continuous time and space: A stochastic control approach. *Journal of Machine Learning Research*, 21 (198):1–34, 2020.
[20] X. Guo, R. Xu, and T. Zariphopoulou. Entropy regularization for mean field games with learning. *Mathematics of Operations research*, 47(4), 2022.

[21] J. Han, R. Hu, and J. Long. Learning high-dimensional McKean-Vlasov forward-backward stochastic differential equations with general distribution dependence. arXiv: 2204.11924, 2022.

[22] K. Hornik, M. Stinchcombe, and H. White. Multilayer feedforward networks are universal approximators. *Neural Networks*, 2:359–366, 1989.

[23] Y. Jia and X. Y. Zhou. Policy evaluation and temporal difference learning in continuous time and space: a martingale approach. to appear in *Journal of Machine learning Research*, 2021.

[24] Y. Jia and X. Y. Zhou. Policy gradient and actor critic learning in continuous time and space: theory and algorithms. to appear in *Journal of Machine learning Research*, 2021. arXiv: 2111.11232v1.

[25] H. Pham and X. Warin. Mean-field neural networks-based algorithms for McKean-Vlasov control problems. arXiv: 2212.11518, 2022.

[26] H. Pham and X. Wei. Dynamic programming for optimal control of stochastic McKean-Vlasov dynamics. *SIAM Journal on Control and Optimization*, 55(2):1069–1101, 2017.

[27] C. Reisinger, W. Stockinger, and Y. Zhang. A fest iterative PDE-based algorithm for feedback controls of nonsmooth mean-field control problems. arXiv: 2108.06740, 2021.

[28] Y. Sun. The exact law of large numbers via fubini extension and characterization of insurable risks. *Journal of Economic Theory*, 126(1):31–69, 2006.

[29] R. Sutton and A. Barto. *Reinforcement Learning: An Introduction*. Cambridge, MA:MIT, 2018.

[30] H. Wang, T. Zariphopoulou, and X. Y. Zhou. Reinforcement learning in continuous time and space: A stochastic control approach. *Journal of Machine Learning Research*, 21(198):1–34, 2020.