SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\nabla_ {\boldsymbol {\mu}} \operatorname {K L} \left(\rho_ {\theta} \| \rho^ {*}\right) = \mathbb {E} _ {\boldsymbol {x} \sim \rho_ {\theta}} \left[ \nabla V (\boldsymbol {x}) \right], \quad \nabla_ {\Sigma} \operatorname {K L} \left(\rho_ {\theta} \| \rho^ {*}\right) = \frac {1}{2} \left(\mathbb {E} _ {\boldsymbol {x} \sim \rho_ {\theta}} \left[ \nabla^ {2} V (\boldsymbol {x}) \right] - \Sigma^ {- 1}\right). \tag {82}
$$
Proof. We compute that
$$
\begin{array}{l} \nabla_ {\boldsymbol {\mu}} \operatorname {K L} (\rho_ {\theta} \| \rho^ {*}) = \nabla_ {\boldsymbol {\mu}} \int \log \frac {\rho_ {\theta} (\boldsymbol {x})}{\rho^ {*} (\boldsymbol {x})} \rho_ {\theta} (\boldsymbol {x}) \mathrm {d} \boldsymbol {x} \\ = \int \frac {\nabla_ {\boldsymbol {\mu}} \rho_ {\theta} (\boldsymbol {x})}{\rho_ {\theta} (\boldsymbol {x})} \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} + \int \log \frac {\rho_ {\theta} (\boldsymbol {x})}{\rho^ {*} (\boldsymbol {x})} \nabla_ {\boldsymbol {\mu}} \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} \\ \stackrel {*} {=} \nabla_ {\boldsymbol {\mu}} \int \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} - \int \log \frac {\rho_ {\theta} (\boldsymbol {x})}{\rho^ {*} (\boldsymbol {x})} \nabla_ {\boldsymbol {x}} \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} \\ = \int \nabla_ {\boldsymbol {x}} \log \frac {\rho_ {\theta} (\boldsymbol {x})}{\rho^ {*} (\boldsymbol {x})} \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} \\ = \int \nabla_ {\boldsymbol {x}} \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} - \int \nabla_ {\boldsymbol {x}} \log \rho^ {*} (\boldsymbol {x}) \cdot \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} \\ = \mathbb {E} _ {\boldsymbol {x} \sim \rho_ {\theta}} [ \nabla V (\boldsymbol {x}) ], \\ \end{array}
$$
where we have used the fact that  $\rho_{\theta}$  is a Gaussian density in  $*$ . Similarly, we have
$$
\begin{array}{l} \nabla_ {\Sigma} \operatorname {K L} \left(\rho_ {\theta} \| \rho^ {*}\right) = \nabla_ {\Sigma} \int \log \frac {\rho_ {\theta} (\boldsymbol {x})}{\rho^ {*} (\boldsymbol {x})} \rho_ {\theta} (\boldsymbol {x}) \mathrm {d} \boldsymbol {x} \\ = \int \frac {\nabla_ {\Sigma} \rho_ {\theta} (\boldsymbol {x})}{\rho_ {\theta} (\boldsymbol {x})} \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} + \int \log \frac {\rho_ {\theta} (\boldsymbol {x})}{\rho^ {*} (\boldsymbol {x})} \nabla_ {\Sigma} \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} \\ \stackrel {(a)} {=} \int \log \frac {\rho_ {\theta} (\boldsymbol {x})}{\rho^ {*} (\boldsymbol {x})} \left(- \frac {1}{2} \Sigma^ {- 1} + \frac {1}{2} \Sigma^ {- 1} (\boldsymbol {x} - \boldsymbol {\mu}) (\boldsymbol {x} - \boldsymbol {\mu}) ^ {\top} \Sigma^ {- 1}\right) \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} \\ \stackrel {(b)} {=} \frac {1}{2} \int \log \frac {\rho_ {\theta} (\boldsymbol {x})}{\rho^ {*} (\boldsymbol {x})} \nabla_ {\boldsymbol {x}} ^ {2} \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} \\ = \frac {1}{2} \int \nabla_ {\boldsymbol {x}} ^ {2} \log \frac {\rho_ {\theta} (\boldsymbol {x})}{\rho^ {*} (\boldsymbol {x})} \cdot \rho_ {\theta} (\boldsymbol {x}) d \boldsymbol {x} \\ = \frac {1}{2} \left(\mathbb {E} _ {\boldsymbol {x} \sim \rho_ {\theta}} \left[ \nabla^ {2} V (\boldsymbol {x}) \right] - \Sigma^ {- 1}\right). \\ \end{array}
$$
Here again in  $(a)$  and  $(b)$  we have used the closed-form expression of Gaussian densities. Thus,
$$
\nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu}, \Sigma) = \mathbb {E} _ {\boldsymbol {x} \sim \rho_ {\theta}} [ \nabla V (\boldsymbol {x}) ], \quad \nabla_ {\Sigma} E (\boldsymbol {\mu}, \Sigma) = \frac {1}{2} \left(\mathbb {E} _ {\boldsymbol {x} \sim \rho_ {\theta}} [ \nabla^ {2} V (\boldsymbol {x}) ] - \Sigma^ {- 1}\right).
$$
Proof of Theorem 4.1. By definition of Gaussian approximate gradient descent, we have that
$$
\dot {\theta} _ {t} = - G _ {\theta_ {t}} ^ {- 1} \left(\nabla_ {\theta_ {t}} E \left(\theta_ {t}\right)\right), \text {w h e r e} E \left(\theta_ {t}\right) = \mathrm {K L} \left(\rho_ {\theta_ {t}} \| \rho^ {*}\right) \text {a n d} \theta = (\boldsymbol {\mu}, \Sigma).
$$
Applying Theorem B.3, we obtain that  $\dot{\theta}_t = -G_{\theta_t}^{-1}(\nabla_{\theta_t}E(\theta_t))$  is given by
$$
\left\{ \begin{array}{l} \dot {\boldsymbol {\mu}} _ {t} = - \left(2 \nabla_ {\Sigma_ {t}} E \left(\boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) \Sigma_ {t} \boldsymbol {\mu} _ {t} + \left(1 + \boldsymbol {\mu} _ {t} ^ {\top} \boldsymbol {\mu} _ {t}\right) \nabla_ {\boldsymbol {\mu} _ {t}} E \left(\boldsymbol {\mu} _ {t}, \Sigma_ {t}\right)\right) \\ \dot {\Sigma} _ {t} = - \Sigma_ {t} \left(2 \Sigma_ {t} \nabla_ {\Sigma_ {t}} E \left(\boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) + \boldsymbol {\mu} _ {t} \nabla_ {\boldsymbol {\mu} _ {t}} ^ {\top} E \left(\boldsymbol {\mu} _ {t}, \Sigma_ {t}\right)\right) - \left(2 \nabla_ {\Sigma_ {t}} E \left(\boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) \Sigma_ {t} + \nabla_ {\boldsymbol {\mu} _ {t}} E \left(\boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) \boldsymbol {\mu} _ {t} ^ {\top}\right) \Sigma_ {t} \end{array} . \right. \tag {83}
$$
Substituting (82) into (83), we get
$$
\left\{ \begin{array}{l} \dot {\boldsymbol {\mu}} _ {t} = \big (I - \mathbb {E} _ {\boldsymbol {x} \sim \rho_ {t}} [ \nabla^ {2} V (\boldsymbol {x}) ] \Sigma_ {t} \big) \boldsymbol {\mu} _ {t} - (1 + \boldsymbol {\mu} _ {t} ^ {\top} \boldsymbol {\mu} _ {t}) \mathbb {E} _ {\boldsymbol {x} \sim \rho_ {t}} [ \nabla V (\boldsymbol {x}) ] \\ \dot {\Sigma} _ {t} = 2 \Sigma_ {t} - \Sigma_ {t} \left(\Sigma_ {t} \mathbb {E} _ {\boldsymbol {x} \sim \rho_ {t}} [ \nabla^ {2} V (\boldsymbol {x}) ] + \boldsymbol {\mu} _ {t} \mathbb {E} _ {\boldsymbol {x} \sim \rho_ {t}} [ \nabla^ {\top} V (\boldsymbol {x}) ]\right) \\ \qquad - \left(\mathbb {E} _ {\boldsymbol {x} \sim \rho_ {t}} [ \nabla^ {2} V (\boldsymbol {x}) ] \Sigma_ {t} + \mathbb {E} _ {\boldsymbol {x} \sim \rho_ {t}} [ \nabla^ {\top} V (\boldsymbol {x}) ] \boldsymbol {\mu} _ {t}\right) \Sigma_ {t} \end{array} \right..
$$
Next we prove the convergence.
$$
\begin{array}{l} \dot {E} \left(\boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) = \operatorname {t r} \left(\nabla_ {\Sigma_ {t}} E \left(\boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) ^ {\top} \dot {\Sigma} _ {t}\right) + \nabla_ {\boldsymbol {\mu} _ {t}} E \left(\boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) ^ {\top} \dot {\boldsymbol {\mu}} _ {t} \\ = - \operatorname {t r} \left(\left(2 \nabla_ {\Sigma} E (\boldsymbol {\mu}, \Sigma) \Sigma + \nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu}, \Sigma) \boldsymbol {\mu} ^ {\top}\right) ^ {\top} \left(2 \nabla_ {\Sigma} E (\boldsymbol {\mu}, \Sigma) \Sigma + \nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu}, \Sigma) \boldsymbol {\mu} ^ {\top}\right)\right) \\ - \nabla_ {\boldsymbol {\mu}} ^ {\top} E (\boldsymbol {\mu}, \Sigma) \nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu}, \Sigma) \leq 0. \\ \end{array}
$$
$$
0 \leq - \int_ {0} ^ {t} \dot {E} (\boldsymbol {\mu} _ {s}, \Sigma_ {s}) \mathrm {d} s = E (\boldsymbol {\mu} _ {0}, \Sigma_ {0}) - E (\boldsymbol {\mu} _ {t}, \Sigma_ {t}) <   \infty ,
$$
we obtain that  $\dot{E} (\pmb{\mu}_t,\Sigma_t)\rightarrow 0$  as  $t\to \infty$  , which means that there exists  $\pmb{\mu}_{\infty},\Sigma_{\infty}$  such that  $\rho_{t}$  converges to  $\rho_{\infty}$ $\rho_{\infty}$  is the density of  $\mathcal{N}(\pmb{\mu}_{\infty},\Sigma_{\infty})$  . (Since  $E(\pmb{\mu}_t,\Sigma_t)$  is given by the KL divergence between  $\rho_{t}$  and  $\rho^*$  , by Lemma K.1 it will diverge if  $\pmb{\mu}_t$  or  $\Sigma_t$  diverges.) In particular, it satisfies that
$$
\left\{ \begin{array}{l} 2 \nabla_ {\Sigma} E (\boldsymbol {\mu} _ {\infty}, \Sigma_ {\infty}) + \nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu} _ {\infty}, \Sigma_ {\infty}) \boldsymbol {\mu} _ {\infty} ^ {\top} = 0 \\ \nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu} _ {\infty}, \Sigma_ {\infty}) = \mathbf {0} \end{array} \right.,
$$
$$
\left\{ \begin{array}{l} \nabla_ {\Sigma} E (\boldsymbol {\mu} _ {\infty}, \Sigma_ {\infty}) = 0 \\ \nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu} _ {\infty}, \Sigma_ {\infty}) = \mathbf {0} \end{array} \right.,
$$
and implies that  $\rho_{\infty} = \rho_{\theta^{*}}$
Lemma K.2. Given  $\alpha$ -strongly convex measure  $\rho^{*}$ , we define  $\theta^{*}$  as the unique minimizer of  $\mathrm{KL}(\rho_{\theta} \parallel \rho^{*})$  where  $\rho_{\theta}$  denotes a Gaussian measure with parameters  $\theta$ . Then it holds that
$$
\begin{array}{l} \left. \left\| \mathbb {E} _ {\rho_ {\theta}} [ \nabla V (\boldsymbol {x}) ] \right\| _ {2} ^ {2} + \operatorname {t r} \left(\left(\mathbb {E} _ {\rho_ {\theta}} [ \nabla^ {2} V (\boldsymbol {x}) ] - \Sigma^ {- 1}\right) \Sigma \left(\mathbb {E} _ {\rho_ {\theta}} [ \nabla^ {2} V (\boldsymbol {x}) ] - \Sigma^ {- 1}\right)\right) \right. \\ \geq 2 \alpha \left(\mathrm {K L} \left(\rho_ {\theta} \| \rho^ {*}\right) - \mathrm {K L} \left(\rho_ {\theta^ {*}} \| \rho^ {*}\right)\right). \\ \end{array}
$$
This is proven in the Appendix D of [43]. The proof idea is to consider the Gaussian approximate Wasserstein gradient flow from  $\rho_{\theta}$  with the target  $\rho^{*}$ .
Lemma K.3. Given  $\alpha$ -strongly convex measure  $\rho^{*}$ , we define  $\theta^{*}$  as the unique minimizer of  $\mathrm{KL}(\rho_{\theta} \| \rho^{*})$  where  $\rho_{\theta}$  denotes a Gaussian measure with parameters  $\theta$ . Then the Wasserstein-2 distance between  $\rho_{\theta}$  and  $\rho_{\theta^{*}}$  satisfies that
$$
\alpha \mathcal {W} _ {2} ^ {2} \left(\rho_ {\theta}, \rho_ {\theta^ {*}}\right) \leq \mathrm {K L} \left(\rho_ {\theta} \| \rho^ {*}\right) - \mathrm {K L} \left(\rho_ {\theta^ {*}} \| \rho^ {*}\right).
$$
This is Lemma E.2 of [13].
Proof of Theorem 4.2. From Lemma K.1 and the proof of Theorem 4.1 we know that
$$
\begin{array}{l} \dot {E} (\boldsymbol {\mu} _ {t}, \Sigma_ {t}) = - \operatorname {t r} \left((2 \nabla_ {\Sigma} E (\boldsymbol {\mu}, \Sigma) \Sigma + \nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu}, \Sigma) \boldsymbol {\mu} ^ {\top}) ^ {\top} (2 \nabla_ {\Sigma} E (\boldsymbol {\mu}, \Sigma) \Sigma + \nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu}, \Sigma) \boldsymbol {\mu} ^ {\top})\right) \\ - \nabla_ {\boldsymbol {\mu}} ^ {\top} E (\boldsymbol {\mu}, \Sigma) \nabla_ {\boldsymbol {\mu}} E (\boldsymbol {\mu}, \Sigma) \\ = - \operatorname {t r} \left(\left(\mathbb {E} _ {\rho_ {\theta}} [ \nabla^ {2} V (\boldsymbol {x}) ] \Sigma - I + \mathbb {E} _ {\rho_ {\theta}} [ \nabla V (\boldsymbol {x}) ] \boldsymbol {\mu} ^ {\top}\right) ^ {\top} \left(\mathbb {E} _ {\rho_ {\theta}} [ \nabla^ {2} V (\boldsymbol {x}) ] \Sigma - I + \mathbb {E} _ {\rho_ {\theta}} [ \nabla V (\boldsymbol {x}) ] \boldsymbol {\mu} ^ {\top}\right)\right) \\ - \left\| \mathbb {E} _ {\rho_ {\theta}} [ \nabla V (\boldsymbol {x}) ] \right\| _ {2} ^ {2}. \\ \end{array}
$$
For convenience we let  $\pmb{\eta}_t = \mathbb{E}_{\rho_\theta}[\nabla V(\pmb{x})]$  and  $S_{t} = (\mathbb{E}_{\rho_{\theta}}[\nabla^{2}V(\pmb {x})] - \Sigma^{-1})\Sigma^{1 / 2}$ . Then we get
$$
\begin{array}{l} - \dot {E} (\boldsymbol {\mu} _ {t}, \Sigma_ {t}) = \left[ \operatorname {v e c} ^ {\top} (S _ {t}) \quad \boldsymbol {\eta} _ {t} \right] \left[ \begin{array}{c c} I _ {d} \otimes \Sigma_ {t} & \boldsymbol {\mu} _ {t} \otimes \Sigma_ {t} ^ {1 / 2} \\ \boldsymbol {\mu} _ {t} ^ {\top} \otimes \Sigma_ {t} ^ {1 / 2} & (1 + \boldsymbol {\mu} _ {t} ^ {\top} \boldsymbol {\mu} _ {t}) I _ {d} \end{array} \right] \left[ \begin{array}{c} \operatorname {v e c} (S _ {t}) \\ \boldsymbol {\eta} _ {t} \end{array} \right] \\ =: \left[ \begin{array}{c c} \operatorname {v e c} ^ {\top} (S _ {t}) & \boldsymbol {\eta} _ {t} \end{array} \right] M _ {t} \left[ \begin{array}{c} \operatorname {v e c} (S _ {t}) \\ \boldsymbol {\eta} _ {t} \end{array} \right]. \\ \end{array}
$$
Noting that  $M_t \to M_\infty$ , for any  $\epsilon > 0$  there exists  $T > 0$  such that the smallest eigenvalue  $\gamma_t / \alpha$  of  $M_t$  satisfies  $\gamma_t \geq \gamma - \epsilon$  for any  $t > T$ , where  $\gamma / \alpha$  is the smallest eigenvalue of  $M_\infty$ .
Moreover, by Lemma K.2 we have
$$
\left[ \operatorname {v e c} ^ {\top} (S _ {t}) \quad \boldsymbol {\eta} _ {t} \right] \left[ \begin{array}{c} \operatorname {v e c} (S _ {t}) \\ \boldsymbol {\eta} _ {t} \end{array} \right] \geq 2 \alpha (\operatorname {K L} (\rho_ {\theta} \| \rho^ {*}) - \operatorname {K L} (\rho_ {\infty} \| \rho^ {*})).
$$
Thus, for  $t > T$  the derivative of KL divergence is controlled, i.e.,
$$
- \partial_ {t} \operatorname {K L} \left(\rho_ {t} \| \rho^ {*}\right) \geq 2 (\gamma - \epsilon) \left(\operatorname {K L} \left(\rho_ {\theta} \| \rho^ {*}\right) - \operatorname {K L} \left(\rho_ {\infty} \| \rho^ {*}\right)\right).
$$
By Gronwall's inequality we know
$$
\mathrm {K L} \left(\rho_ {\theta} \| \rho^ {*}\right) - \mathrm {K L} \left(\rho_ {\infty} \| \rho^ {*}\right) = \mathcal {O} \left(e ^ {- 2 (\gamma - \epsilon) t}\right).
$$
By Lemma K.3 this implies
$$
\mathcal {W} _ {2} ^ {2} \left(\rho_ {\theta}, \rho_ {\theta^ {*}}\right) = \mathcal {O} \left(e ^ {- 2 (\gamma - \epsilon) t}\right).
$$
$$
\mathcal {W} _ {2} ^ {2} \left(\rho_ {\theta}, \rho_ {\theta^ {*}}\right) = \| \boldsymbol {\mu} - \boldsymbol {\mu} ^ {*} \| _ {2} ^ {2} + \operatorname {t r} \left(\left(\Sigma^ {1 / 2} - \left(\Sigma^ {*}\right) ^ {1 / 2}\right) ^ {2}\right),
$$
$$
\left\| \boldsymbol {\mu} _ {t} - \boldsymbol {\mu} ^ {*} \right\| = \mathcal {O} \left(e ^ {- (\gamma - \epsilon) t}\right), \quad \left\| \Sigma_ {t} - \Sigma^ {*} \right\| = \mathcal {O} \left(e ^ {- (\gamma - \epsilon) t}\right), \quad \forall \epsilon > 0.
$$
Finally, we provide a lower bound on  $\gamma$ . Note that for any  $u > 0$  if  $\pmb{\mu}^{*}\neq \mathbf{0}$  we have
$$
\left[ \begin{array}{c c} \frac {1}{1 + u} I _ {d} \otimes \Sigma^ {*} & \boldsymbol {\mu} ^ {*} \otimes (\Sigma^ {*}) ^ {1 / 2} \\ \boldsymbol {\mu} ^ {* ^ {\top}} \otimes (\Sigma^ {*}) ^ {1 / 2} & (1 + u) \boldsymbol {\mu} ^ {* ^ {\top}} \boldsymbol {\mu} ^ {*} I _ {d} \end{array} \right] \succeq 0.
$$
$$
\left[ \begin{array}{c c} I _ {d} \otimes \Sigma^ {*} & \boldsymbol {\mu} ^ {*} \otimes (\Sigma^ {*}) ^ {1 / 2} \\ \boldsymbol {\mu} ^ {* ^ {\top}} \otimes (\Sigma^ {*}) ^ {1 / 2} & (1 + \boldsymbol {\mu} ^ {* ^ {\top}} \boldsymbol {\mu} ^ {*}) I _ {d} \end{array} \right] \succeq \left[ \begin{array}{c c} \frac {u}{1 + u} I _ {d} \otimes \Sigma^ {*} & \\ & (1 - u \boldsymbol {\mu} ^ {* ^ {\top}} \boldsymbol {\mu} ^ {*}) I _ {d} \end{array} \right] =: \Omega_ {u}.
$$
Since  $\Sigma^{*}$  satisfies that
$$
\mathbb {E} _ {\rho_ {\theta^ {*}}} \left[ \nabla^ {2} V (\boldsymbol {x}) \right] - \left(\Sigma^ {*}\right) ^ {- 1} = 0,
$$
and  $\nabla^2 V(\pmb{x})\preceq \beta I_d$  , we know the smallest eigenvalue of  $\Sigma^{*}$  is at least  $1 / \beta$  . Thus, the smallest eigenvalue of  $\Omega_{u}$  is
$$
\min  \left\{\frac {u}{\beta (1 + u)}, 1 - u r \right\}, \text {w h e r e} u > 0, r = \boldsymbol {\mu} ^ {* \top} \boldsymbol {\mu} ^ {*}.
$$
$$
\frac {\gamma}{\alpha} \geq \max _ {u > 0} \min \left\{\frac {u}{\beta (1 + u)}, 1 - u r \right\} = \frac {2}{1 + \beta (1 + r) + \sqrt {(1 + \beta (1 + r)) ^ {2} - 4 \beta}} > \frac {1}{\beta (1 + r) + 1}.
$$
If  $\pmb{\mu}^{*} = 0$ , we still have
$$
\frac {\gamma}{\alpha} = \min  \left\{\frac {1}{\beta}, 1 \right\} > \frac {1}{\beta + 1}.
$$
L Proofs of Uniform in Time Convergence
To show Theorem 3.7 we need a lemma on the convergence of empirical measures in the i.i.d. setting. There are results for general measures on this given by [25, 47, 24] but for our purpose we always have a Gaussian distributions as the limit and there are tight results with faster convergence rates as shown in [9, 45, 46]:
Lemma L.1 (Convergence of empirical measures for Gaussian distributions). Fix the dimension  $d \geq 1$ . There exists a constant  $C_d$  such that for all  $N \geq 1$ , with  $\mu_N = \frac{1}{N} \sum_{k=1}^N \delta_{X_k}$  where  $\{X_k\}$  is i.i.d. sequence drawn from  $\mu \sim \mathcal{N}(\mathbf{0}, I_d)$ , we have
$$
\mathbb {E} \left[ \mathcal {W} _ {2} ^ {2} \left(\mu_ {N}, \mu\right) \right] \leq C _ {d} \times \left\{ \begin{array}{c c} N ^ {- 1} \log \log N & i f d = 1 \\ N ^ {- 1} (\log N) ^ {2} & i f d = 2 \\ N ^ {- 2 / d} & i f d \geq 3 \end{array} \right..
$$
Proof of Theorem 3.7. Suppose the sample mean and covariance at time  $t$  is  $\pmb{m}_t$  and  $C_t$ , and that the mean and covariance of the mean-field limit is  $\pmb{\mu}_t$  and  $\Sigma_t$ . We bound  $\mathbb{E}[\mathcal{W}_2^2 (\zeta_N^{(t)},\rho_t)]$  using Theorems 3.1 and 3.6 and Lemma L.1 in six steps.
Step I. We prove that  $(\pmb{x}_i^{(t)})$  has the same distribution as  $\left(\widetilde{\pmb{x}}_i^{(t)}\right)$  where  $\widetilde{\pmb{x}}_i^{(t)} = C_t^{1/2}C_0^{-1/2}(\pmb{x}_i^{(0)} - \pmb{m}_0) + \pmb{m}_t$ . Note that according to Theorem 3.6, where we have  $\pmb{x}_i^{(t)} = A_t(\pmb{x}_i^{(0)} - \pmb{m}_0) + \pmb{m}_t$ . Here  $A_t$  is the unique (matrix) solution of the linear system
$$
\dot {A} _ {t} = \left(I - Q ^ {- 1} \left(C _ {t} + \boldsymbol {\mu} _ {t} \boldsymbol {\mu} _ {t} ^ {\top}\right) + Q ^ {- 1} \boldsymbol {b} \boldsymbol {\mu} _ {t} ^ {\top}\right) A _ {t}, \quad A _ {0} = I, \tag {84}
$$
and  $m_t$  and  $C_t$  are the unique solution of the ODE system
$$
\left\{ \begin{array}{l} \dot {\boldsymbol {m}} _ {t} = \left(I - Q ^ {- 1} C _ {t}\right) \boldsymbol {m} _ {t} - \left(1 + \boldsymbol {m} _ {t} ^ {\top} \boldsymbol {m} _ {t}\right) Q ^ {- 1} \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) \\ \dot {C} _ {t} = 2 C _ {t} - C _ {t} \left(C _ {t} + \boldsymbol {m} _ {t} \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) ^ {\top}\right) Q ^ {- 1} - Q ^ {- 1} \left(C _ {t} + \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) \boldsymbol {m} _ {t} ^ {\top}\right) C _ {t}. \end{array} \right. \tag {85}
$$
Since  $C_t$  is the sample covariance, we have
$$
A _ {t} C _ {0} A _ {t} ^ {\top} = C _ {t},
$$
$$
\left(C _ {t} ^ {- 1 / 2} A _ {t} C _ {0} ^ {1 / 2}\right) \left(C _ {t} ^ {- 1 / 2} A _ {t} C _ {0} ^ {1 / 2}\right) ^ {\top} = I.
$$
Thus,  $P_{t} = C_{t}^{-1 / 2}A_{t}C_{0}^{1 / 2}$  is an orthogonal matrix, and we have
$$
A _ {t} = C _ {t} ^ {1 / 2} P _ {t} C _ {0} ^ {- 1 / 2}.
$$
Since the multivariate Gaussian distribution  $\mathcal{N}(\mathbf{0}, I_d)$  is invariant under orthogonal transformation, the joint distribution of  $\left(C_0^{-1/2} (\boldsymbol{x}_i^{(0)} - \boldsymbol{m}_0)\right)_{i=1}^N$  is also invariant under  $P_t$ . Thus, we have  $(\boldsymbol{x}_i^{(t)} - \boldsymbol{m}_t)$  has the same distribution as  $(\widetilde{\boldsymbol{x}}_i^{(t)} - \boldsymbol{m}_t)$  and Step I is proven.
Step II. Establish uniform decay rates for  $\| C_t - Q\| _F$  and  $\| \pmb {m}_t - \pmb {b}\|$ . We begin by checking the energy function
$$
0 \leq E (\boldsymbol {m} _ {t}, C _ {t}) = \frac {1}{2} \left(\operatorname {t r} (Q ^ {- 1} C _ {t}) - \log \det  (Q ^ {- 1} C _ {t}) - d + (\boldsymbol {m} _ {t} - \boldsymbol {b}) ^ {\top} Q ^ {- 1} (\boldsymbol {m} _ {t} - \boldsymbol {b})\right).
$$
As shown in the proof of Theorem 3.1, we have
$$
\dot {E} (\boldsymbol {m} _ {t}, C _ {t}) = - \left\| C _ {t} Q ^ {- 1} - I + \boldsymbol {m} _ {t} (\boldsymbol {m} _ {t} - \boldsymbol {b}) ^ {\top} Q ^ {- 1} \right\| _ {F} ^ {2} - \| Q ^ {- 1} (\boldsymbol {m} _ {t} - \boldsymbol {b}) \| ^ {2} \leq 0.
$$
Thus,  $E(\pmb{m}_t, C_t) \leq E(\pmb{m}_0, C_0)$  for any  $t \geq 0$ . Furthermore, similar to the proof of Theorem 3.1 we check that
$$
\begin{array}{l} - \dot {E} (\boldsymbol {m} _ {t}, C _ {t}) \\ = \left[ \operatorname {v e c} ^ {\top} \left(Q ^ {- 1} C _ {t} - I\right) \quad \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) ^ {\top} Q ^ {- 1} \right] \left[ \begin{array}{c c} I _ {d ^ {2}} & \boldsymbol {m} _ {t} \otimes I _ {d} \\ \boldsymbol {m} _ {t} ^ {\top} \otimes I _ {d} & (1 + \boldsymbol {m} _ {t} ^ {\top} \boldsymbol {m} _ {t}) I _ {d} \end{array} \right] \left[ \begin{array}{c} \operatorname {v e c} \left(Q ^ {- 1} C _ {t} ^ {- 1} - I\right) \\ Q ^ {- 1} (\boldsymbol {m} _ {t} - \boldsymbol {b}) \end{array} \right] \\ \geq \gamma_ {t} \left[ \operatorname {v e c} ^ {\top} \left(Q ^ {- 1} C _ {t} - I\right) \quad \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) ^ {\top} Q ^ {- 1} \right] \left[ \begin{array}{c c} I _ {d ^ {2}} & \\ & 2 Q \end{array} \right] \left[ \begin{array}{c} \operatorname {v e c} \left(Q ^ {- 1} C _ {t} - I\right) \\ Q ^ {- 1} (\boldsymbol {m} _ {t} - \boldsymbol {b}) \end{array} \right] \\ = \gamma_ {t} \operatorname {t r} \left(\left(Q ^ {- 1} C _ {t} - I\right) ^ {\top} \left(Q ^ {- 1} C _ {t} - I\right)\right) + 2 \gamma_ {t} \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) ^ {\top} Q \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) \\ \geq 2 \gamma_ {t} \cdot \left(\operatorname {t r} \left(Q ^ {- 1} C _ {t} - I\right) - \log \det  \left(Q ^ {- 1} C _ {t}\right) + \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) ^ {\top} Q (\boldsymbol {m} _ {t} - \boldsymbol {b})\right) \\ = 4 \gamma_ {t} E \left(\boldsymbol {m} _ {t}, C _ {t}\right). \\ \end{array}
$$
where  $\gamma_{t}$  is the smallest eigenvalue of
$$
\left[ \begin{array}{c c} I _ {d ^ {2}} & \frac {1}{\sqrt {2}} \boldsymbol {m} _ {t} \otimes Q ^ {- 1 / 2} \\ \frac {1}{\sqrt {2}} \boldsymbol {m} _ {t} ^ {\top} \otimes Q ^ {- 1 / 2} & \frac {1}{2} (1 + \boldsymbol {m} _ {t} ^ {\top} \boldsymbol {m} _ {t}) Q ^ {- 1} \end{array} \right],
$$
and as shown in the proof of Theorem 3.1 it has a lower bound
$$
\gamma_ {t} > \frac {1}{1 + \boldsymbol {m} _ {t} ^ {\top} \boldsymbol {m} _ {t} + q _ {\max }},
$$
where  $q_{\mathrm{max}}$  is the largest eigenvalue of  $Q$ . Now since
$$
\frac {1}{2} \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) ^ {\top} Q ^ {- 1} (\boldsymbol {m} - \boldsymbol {b}) \leq E \left(\boldsymbol {m} _ {t}, C _ {t}\right) \leq E \left(\boldsymbol {m} _ {0}, C _ {0}\right),
$$
we know  $(\pmb{m}_t - \pmb{b})^\top (\pmb{m}_t - \pmb{b}) \leq 2q_{\max}E(\pmb{m}_0, C_0)$ . Thus,  $\|\pmb{m}_t\|$  is upper bounded by some quantity  $F_1 = F_{1;Q,\pmb{b},C_0,\pmb{m}_0}$ . Hence  $\gamma_t$  is uniformly lower bounded by
$$
\gamma^ {*} := \inf  _ {t \geq 0} \gamma_ {t} \geq \frac {1}{1 + F _ {1} ^ {2} + q _ {\max}}.
$$
Thus, by Gronwall's inequality we have  $E(\pmb{m}_t, C_t) \leq e^{-4\gamma^* t} E(\pmb{m}_0, C_0)$ . There exists  $F_2 = F_{2;Q,\pmb{b},C_0,\pmb{m}_0}$  such that  $\| \pmb{m}_t - \pmb{b} \| \leq e^{-2\gamma^* t} F_2$ .
Now similarly
$$
0 \leq \frac {1}{2} \left(\operatorname {t r} \left(Q ^ {- 1} C _ {t} - 1\right) - \log \det  \left(Q ^ {- 1} C _ {t}\right)\right) \leq E (\boldsymbol {m} _ {t}, C _ {t}) \leq e ^ {- 4 \gamma^ {*} t} E (\boldsymbol {m} _ {0}, C _ {0})
$$
also renders an upper bound for  $\| C_t - Q \|_F$  with exponential decay noting that  $\operatorname{tr}(A) - \log \det(I + A)$  is quadratic in  $\| A \|_F$  when  $\| A \|_F$  is small, i.e., there exists  $F_3 = F_{3;Q,\mathbf{b},C_0,\mathbf{m}_0}$  such that
$$
\left\| C _ {t} - Q \right\| _ {F} \leq e ^ {- 2 \gamma^ {*} t} F _ {3}.
$$
By Theorem 3.1 we know that  $\pmb{\mu}_t, \Sigma_t$  satisfy the same ODEs as  $\pmb{m}_t, C_t$ :
$$
\left\{ \begin{array}{l} \dot {\boldsymbol {\mu}} _ {t} = (I - Q ^ {- 1} \Sigma_ {t}) \boldsymbol {\mu} _ {t} - (1 + \boldsymbol {\mu} _ {t} ^ {\top} \boldsymbol {\mu} _ {t}) Q ^ {- 1} (\boldsymbol {\mu} _ {t} - \boldsymbol {b}) \\ \dot {\Sigma} _ {t} = 2 \Sigma_ {t} - \Sigma_ {t} \left(\Sigma_ {t} + \boldsymbol {\mu} _ {t} (\boldsymbol {\mu} _ {t} - \boldsymbol {b}) ^ {\top}\right) Q ^ {- 1} - Q ^ {- 1} \left(\Sigma_ {t} + (\boldsymbol {\mu} _ {t} - \boldsymbol {b}) \boldsymbol {\mu} _ {t} ^ {\top}\right) \Sigma_ {t}. \end{array} \right. \tag {86}
$$
Thus, similarly we have
$$
\left\| \boldsymbol {\mu} _ {t} - \boldsymbol {b} \right\| \leq e ^ {- 2 \gamma^ {*} t} F _ {2} ^ {\prime}, \quad \left\| \Sigma_ {t} - Q \right\| _ {F} \leq e ^ {- 2 \gamma^ {*} t} F _ {3} ^ {\prime}.
$$
Step III. Show that  $\| C_t - \Sigma_t\| _F$  and  $\| \pmb {m}_t - \pmb {\mu}_t\|$  can be controlled after sufficient time.
For any  $\epsilon > 0$  we define