SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\Theta_ {\epsilon} := \left\{\left(\boldsymbol {v}, S\right) \in \mathbb {R} ^ {d} \times \operatorname {S y m} ^ {+} (d, \mathbb {R}): \| \boldsymbol {v} - \boldsymbol {b} \| \leq \epsilon , \text {a n d} (1 - \epsilon) Q \preceq S \preceq (1 + \epsilon) Q \right\},
$$
and consider the relative energy function
$$
E \left(\boldsymbol {m} _ {t}, C _ {t}, \boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) = \frac {1}{2} \left(\operatorname {t r} \left(C _ {t} ^ {- 1} \Sigma_ {t} - I\right) - \log \det  \left(C _ {t} ^ {- 1} \Sigma_ {t}\right) + \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 1} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right)\right).
$$
We show that for  $\epsilon$  small enough if  $(\pmb{m}_t,C_t),(\pmb {\mu}_t,\Sigma_t)\in \Theta_\epsilon$  , then
$$
\dot {E} \left(\boldsymbol {m} _ {t}, C _ {t}, \boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) \leq 0.
$$
We derive that
$$
\begin{array}{l} \dot {E} \left(\boldsymbol {m} _ {t}, C _ {t}, \boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) \\ = \frac {1}{2} \operatorname {t r} \left(C _ {t} ^ {- 1} \left(\Sigma_ {t} - C _ {t}\right) \left(\Sigma_ {t} ^ {- 1} \dot {\Sigma} _ {t} - C _ {t} ^ {- 1} \dot {C} _ {t}\right)\right) + \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 1} \left(\dot {\boldsymbol {m}} _ {t} - \dot {\boldsymbol {\mu}} _ {t}\right). \\ \end{array}
$$
Here note that
$$
\begin{array}{l} - C _ {t} ^ {- 1} \left(\Sigma_ {t} - C _ {t}\right) \left(\Sigma_ {t} ^ {- 1} \dot {\Sigma} _ {t} - C _ {t} ^ {- 1} \dot {C} _ {t}\right) \\ \stackrel {\mathrm {t r}} {=} C _ {t} ^ {- 1} \left(\Sigma_ {t} - C _ {t}\right) \left(\left(\Sigma_ {t} + \boldsymbol {\mu} _ {t} \left(\boldsymbol {\mu} _ {t} - \boldsymbol {b}\right) ^ {\top} - C _ {t} - \boldsymbol {m} _ {t} \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) ^ {\top}\right) Q ^ {- 1} \right. \\ + \Sigma_ {t} ^ {- 1} Q ^ {- 1} (\Sigma_ {t} + (\boldsymbol {\mu} _ {t} - \boldsymbol {b}) \boldsymbol {\mu} _ {t} ^ {\top}) \Sigma_ {t} - C _ {t} ^ {- 1} Q ^ {- 1} (C _ {t} + (\boldsymbol {m} _ {t} - \boldsymbol {b}) \boldsymbol {m} _ {t} ^ {\top}) C _ {t}) \\ \stackrel {\text {二}} {=} 2 \left(C _ {t} - \Sigma_ {t}\right) ^ {2} Q ^ {- 1} C _ {t} ^ {- 1} \\ + 2 \left(C _ {t} - \Sigma_ {t}\right) \left(\boldsymbol {m} _ {t} \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) ^ {\top} - \boldsymbol {\mu} _ {t} \left(\boldsymbol {\mu} _ {t} - \boldsymbol {b}\right) ^ {\top}\right) Q ^ {- 1} C _ {t} ^ {- 1} \\ \geq \left(C _ {t} - \Sigma_ {t}\right) ^ {2} Q ^ {- 1} C _ {t} ^ {- 1} \\ \left. - \left(\boldsymbol {m} _ {t} (\boldsymbol {m} _ {t} - \boldsymbol {b}) ^ {\top} - \boldsymbol {\mu} _ {t} (\boldsymbol {\mu} _ {t} - \boldsymbol {b}) ^ {\top}\right) ^ {2} Q ^ {- 1} C _ {t} ^ {- 1} \right. \\ \geq \left(C _ {t} - \Sigma_ {t}\right) ^ {2} Q ^ {- 1} C _ {t} ^ {- 1} \\ - \frac {1}{1 - \epsilon} \left(\boldsymbol {\mu} _ {t} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} + \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right) ^ {\top}\right) ^ {2} Q ^ {- 2}. \\ \end{array}
$$
$$
\begin{array}{l} - \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 1} \left(\dot {\boldsymbol {m}} _ {t} - \dot {\boldsymbol {\mu}} _ {t}\right) \\ = \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} \left(Q ^ {- 2} - Q ^ {- 1}\right) \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) + \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 2} \left(C _ {t} \boldsymbol {m} _ {t} - \Sigma_ {t} \boldsymbol {\mu} _ {t}\right) \\ - \left(\boldsymbol {m} _ {t} ^ {\top} \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} ^ {\top} \boldsymbol {\mu} _ {t}\right) \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 2} \boldsymbol {b} + \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 2} \left(\boldsymbol {m} _ {t} \boldsymbol {m} _ {t} ^ {\top} \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} \boldsymbol {\mu} _ {t} ^ {\top} \boldsymbol {\mu} _ {t}\right) \\ =: I _ {1} + I _ {2} + I _ {3} + I _ {4}. \\ \end{array}
$$
Here we have
$$
\begin{array}{l} I _ {2} = \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 2} C _ {t} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) + \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 2} \left(C _ {t} - \Sigma_ {t}\right) \boldsymbol {\mu} _ {t} \\ \geq (1 - \epsilon) (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) ^ {\top} Q ^ {- 1} (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) + (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) ^ {\top} Q ^ {- 2} (C _ {t} - \Sigma_ {t}) \boldsymbol {\mu} _ {t} \\ \geq (1 - \epsilon) \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 1} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) - \frac {1}{2} \operatorname {t r} \left(\left(C _ {t} - \Sigma_ {t}\right) ^ {2} Q ^ {- 1} C _ {t} ^ {- 1}\right) \\ - \frac {1}{2} \operatorname {t r} \left(\left(\boldsymbol {\mu} _ {t} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top}\right) ^ {2} Q ^ {- 3} C _ {t}\right) \\ \geq (1 - \epsilon) (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) ^ {\top} Q ^ {- 1} (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) - \frac {1}{2} \operatorname {t r} \left((C _ {t} - \Sigma_ {t}) ^ {2} Q ^ {- 1} C _ {t} ^ {- 1}\right) \\ - \frac {1}{2} (1 + \epsilon) \operatorname {t r} \left(\left(\boldsymbol {\mu} _ {t} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top}\right) ^ {2} Q ^ {- 2}\right), \\ \end{array}
$$
$$
I _ {3} + I _ {4} = \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 2} \left(\boldsymbol {\mu} _ {t} \boldsymbol {\mu} _ {t} ^ {\top} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) + \left(\boldsymbol {m} _ {t} \boldsymbol {m} ^ {\top} - \boldsymbol {\mu} _ {t} \boldsymbol {\mu} _ {t} ^ {\top}\right) \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right)\right).
$$
Combining all these together we have
$$
\begin{array}{l} - \dot {E} \left(\boldsymbol {m} _ {t}, C _ {t}, \boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) \\ \geq \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} \left(Q ^ {- 2} - \epsilon Q ^ {- 1}\right) \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) - \frac {1}{2} (1 + \epsilon) \operatorname {t r} \left(\left(\boldsymbol {\mu} _ {t} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top}\right) ^ {2} Q ^ {- 2}\right) \\ - \frac {1}{2 (1 - \epsilon)} \operatorname {t r} \left(\left(\boldsymbol {\mu} _ {t} (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) ^ {\top} + (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) (\boldsymbol {m} _ {t} - \boldsymbol {b}) ^ {\top}\right) ^ {2} Q ^ {- 2}\right) \\ + \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 2} \left(\boldsymbol {\mu} _ {t} \boldsymbol {\mu} _ {t} ^ {\top} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) + \left(\boldsymbol {m} _ {t} \boldsymbol {m} ^ {\top} - \boldsymbol {\mu} _ {t} \boldsymbol {\mu} _ {t} ^ {\top}\right) \left(\boldsymbol {m} _ {t} - \boldsymbol {b}\right)\right) \\ = (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) ^ {\top} \left(Q ^ {- 2} - \epsilon Q ^ {- 1} - \frac {\epsilon (2 - \epsilon)}{2 (1 - \epsilon)} Q ^ {- 2} \boldsymbol {\mu} _ {t} \boldsymbol {\mu} _ {t} ^ {\top}\right) (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) + \mathcal {O} (\epsilon^ {2}). \\ \end{array}
$$
Since  $0 < q_{\min}I \preceq Q \preceq q_{\max}I$ , and  $\| \pmb{m}_t\|$  and  $\| \pmb{\mu}_t\|$  are bounded by  $F_1$  as shown in Step II, there exists  $\epsilon_0 = \epsilon_{Q,\pmb{b},C_0,\pmb{m}_0,\Sigma_0,\pmb{\mu}_0}$  ( $\epsilon_0$  can be seen as a continuous function) such that for any  $\epsilon \leq \epsilon_0$  we have  $\dot{E}(\pmb{m}_t,C_t,\pmb{\mu}_t,\Sigma_t) \leq 0$  as long as we have  $(\pmb{m}_t,C_t) \in \Theta_\epsilon$  and  $(\pmb{\mu}_t,\Sigma_t) \in \Theta_\epsilon$ .
Given  $Q, B, C_0, \pmb{m}_t, \Sigma_0, \pmb{\mu}_0$  suppose at time  $t_0$  we have  $(\pmb{m}_{t_0}, C_{t_0}) \in \Theta_{\epsilon_0}$  and  $(\pmb{\mu}_{t_0}, \Sigma_{t_0}) \in \Theta_{\epsilon_0}$ . Then for any  $t > t_0$  we know
$$
\begin{array}{l} E \left(\boldsymbol {m} _ {t}, C _ {t}, \boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) \leq E \left(\boldsymbol {m} _ {t _ {0}}, C _ {t _ {0}}, \boldsymbol {\mu} _ {t _ {0}}, \Sigma_ {t _ {0}}\right) \\ \leq \frac {1}{4} \operatorname {t r} \left(\left(C _ {t _ {0}} - \Sigma_ {t _ {0}}\right) ^ {2} C _ {t _ {0}} ^ {- 2}\right) + \left(\boldsymbol {m} _ {t _ {0}} - \boldsymbol {\mu} _ {t _ {0}}\right) ^ {\top} Q ^ {- 1} \left(\boldsymbol {m} _ {t _ {0}} - \boldsymbol {\mu} _ {t _ {0}}\right) \\ \leq \frac {1}{4 (1 - \epsilon_ {0}) ^ {2} q _ {\operatorname* {m i n}} ^ {2}} \| C _ {t _ {0}} - \Sigma_ {t _ {0}} \| _ {F} ^ {2} + \frac {1}{q _ {\operatorname* {m i n}}} \| \boldsymbol {m} _ {t _ {0}} - \boldsymbol {\mu} _ {t _ {0}} \| ^ {2}. \\ \end{array}
$$
$$
\lim_{\| A\|_{F}\to 0}\frac{\operatorname{tr}(A) - \log\det(I + A)}{\operatorname{tr}(A^{\top}A)} = \frac{1}{2}.
$$
Fixing any  $\delta > 0$  as long as  $\epsilon_0$  is small enough we have
$$
\begin{array}{l} E \left(\boldsymbol {m} _ {t}, C _ {t}, \boldsymbol {\mu} _ {t}, \Sigma_ {t}\right) \geq \frac {1}{4 + \delta} \operatorname {t r} \left(\left(C _ {t} - \Sigma_ {t}\right) ^ {2} C _ {t} ^ {- 2}\right) + \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 1} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) \\ \geq \frac {1}{(4 + \delta) (1 + \epsilon_ {0}) ^ {2} q _ {\max} ^ {2}} \| C _ {t} - \Sigma_ {t} \| _ {F} ^ {2} + \frac {1}{q _ {\max}} \| \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} \| ^ {2}. \\ \end{array}
$$
Therefore, we conclude that for any given  $Q$ ,  $b$ ,  $C_0$ ,  $\pmb{m}_0$ ,  $\Sigma_0$  and  $\pmb{\mu}_0$  there exists  $\epsilon_0$  as stated above and  $F_{4} = F_{4;Q,b,C_{0},\pmb{m}_{0},\Sigma_{0},\pmb{\mu}_{0}} > 0$  such that as long as  $(\pmb{m}_{t_0},C_{t_0})\in \Theta_{\epsilon_0}$  and  $(\pmb{\mu}_{t_0},\Sigma_{t_0})\in \Theta_{\epsilon_0}$  then for any  $t > t_0$
$$
\begin{array}{l} \left\| C _ {t} - \Sigma_ {t} \right\| _ {F} ^ {2} \leq F _ {4} \left(\left\| C _ {t _ {0}} - \Sigma_ {t _ {0}} \right\| _ {F} ^ {2} + \left\| \boldsymbol {m} _ {t _ {0}} - \boldsymbol {\mu} _ {t _ {0}} \right\| ^ {2}\right), \\ \left\| \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} \right\| ^ {2} \leq F _ {4} \left(\left\| C _ {t _ {0}} - \Sigma_ {t _ {0}} \right\| _ {F} ^ {2} + \left\| \boldsymbol {m} _ {t _ {0}} - \boldsymbol {\mu} _ {t _ {0}} \right\| ^ {2}\right). \\ \end{array}
$$
Step IV. Uniformly bound  $\| C_t - \Sigma_t\| _F^2$  and  $\| \pmb {m}_t - \pmb {\mu}_t\| ^2$  using  $\| C_0 - \Sigma_0\| _F^2 +\| \pmb {m}_0 - \pmb {\mu}_0\| ^2$
Note that by definition of the Frobenius norm (or any matrix norm), given  $\epsilon_0 > 0$  there exists  $\epsilon_1 \in (0, \epsilon_0)$  such that for any  $S \in \operatorname{Sym}(d, \mathbb{R})$  as long as the norm is small enough, i.e.,  $\| S - Q\|_F \leq \epsilon_1$ , then we have  $(1 - \epsilon_0)Q \preceq S \preceq (1 + \epsilon_0)Q$ . Then by Step II we know that if we set  $F_5 = \max \{F_2, F_3, F_2', F_3'\}$  and  $t_0 > -\frac{1}{2\gamma^*} \log \frac{\epsilon_1}{F_5}$  then the following bounds hold:
$$
\left\| \boldsymbol {m} _ {t} - \boldsymbol {b} \right\| <   \epsilon_ {1}, \quad \left\| C _ {t} - Q \right\| _ {F} <   \epsilon_ {1}, \quad \left\| \boldsymbol {\mu} _ {t} - \boldsymbol {b} \right\| <   \epsilon_ {1}, \quad \left\| \Sigma_ {t} - Q \right\| _ {F} <   \epsilon_ {1}.
$$
Now it is straightforward to check from (85) and (86) and the results in Step II that there exists  $F_{6} = F_{6;Q,b,C_{0},m_{0},\Sigma_{0},\mu_{t}}$  such that for any  $t \geq 0$
$$
\frac {\mathrm {d}}{\mathrm {d} t} \| \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} \| ^ {2} \leq F _ {6} \| \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} \|, \quad \frac {\mathrm {d}}{\mathrm {d} t} \| C _ {t} - \Sigma_ {t} \| _ {F} ^ {2} \leq F _ {6} \| C _ {t} - \Sigma_ {t} \| _ {F} ^ {2}.
$$
Thus, by Gronwall's inequality we have
$$
\left\| \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} \right\| ^ {2} \leq e ^ {F _ {6} t} \left\| \boldsymbol {m} _ {0} - \boldsymbol {\mu} _ {0} \right\| ^ {2}, \quad \left\| C _ {t} - \Sigma_ {t} \right\| _ {F} ^ {2} \leq e ^ {F _ {6} t} \left\| C _ {0} - \Sigma_ {0} \right\| _ {F} ^ {2}.
$$
Combining this with Step III, we know for any  $t \geq 0$ , there exists  $F_7 = e^{F_6 t_0} F_4$  (only depending on  $Q, b, C_0, m_0, \Sigma_0, \mu_0$ ) such that
$$
\begin{array}{l} \left\| C _ {t} - \Sigma_ {t} \right\| _ {F} ^ {2} \leq F _ {7} \left(\left\| C _ {0} - \Sigma_ {0} \right\| _ {F} ^ {2} + \left\| \boldsymbol {m} _ {0} - \boldsymbol {\mu} _ {0} \right\| ^ {2}\right), \\ \left\| \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} \right\| ^ {2} \leq F _ {7} \left(\left\| C _ {0} - \Sigma_ {0} \right\| _ {F} ^ {2} + \left\| \boldsymbol {m} _ {0} - \boldsymbol {\mu} _ {0} \right\| ^ {2}\right). \\ \end{array}
$$
Step V. Let  $\pmb{y}_i^{(t)}\coloneqq \Sigma_t^{1 / 2}\Sigma_0^{-1 / 2}(\pmb{x}_i^{(0)} - \pmb{\mu}_0) + \pmb{\mu}_t$ . Define  $\xi_N^{(t)} = \frac{1}{N}\sum_{i = 1}^N\delta_{\pmb{y}_i^{(t)}}$  and  $\widetilde{\zeta}_N^{(t)} = \frac{1}{N}\sum_{i = 1}^N\delta_{\widetilde{\pmb{x}}_i^{(t)}}$ . We show that
$$
\mathbb {E} \left[ \mathcal {W} _ {2} ^ {2} \left(\xi_ {N} ^ {(t)}, \widetilde {\zeta} _ {N} ^ {(t)}\right) \right] \stackrel {{\diamond}} {{\leq}} \mathbb {E} \left[ \frac {1}{N} \sum_ {i = 1} ^ {N} \left\| \widetilde {\boldsymbol {x}} _ {N} ^ {(t)} - \boldsymbol {y} _ {N} ^ {(t)} \right\| ^ {2} \right] \stackrel {{\diamond}} {{=}} o \left(\frac {\log \log N}{N}\right).
$$
Since  $\circ$  is trivial by the definition of the Wasserstein metric, we only need to check  $\diamond$ . In fact,
$$
\begin{array}{l} \frac {1}{N} \sum_ {i = 1} ^ {N} \left\| \widetilde {\boldsymbol {x}} _ {N} ^ {(t)} - \boldsymbol {y} _ {N} ^ {(t)} \right\| ^ {2} \\ \leq \frac {3}{N} \sum_ {i = 1} ^ {N} \left\| \left(\Sigma_ {t} ^ {1 / 2} \Sigma_ {0} ^ {- 1 / 2} - C _ {t} ^ {1 / 2} C _ {0} ^ {- 1 / 2}\right) \left(\boldsymbol {x} _ {i} ^ {(0)} - \boldsymbol {\mu} _ {0}\right) \right\| ^ {2} \\ + 3 \left\| C _ {t} ^ {1 / 2} C _ {0} ^ {- 1 / 2} \left(\boldsymbol {m} _ {0} - \boldsymbol {\mu} _ {0}\right) \right\| ^ {2} + 3 \| \boldsymbol {\mu} _ {t} - \boldsymbol {m} _ {t} \| ^ {2} \\ = 3 \left\| \Sigma_ {t} ^ {1 / 2} \Sigma_ {0} ^ {- 1 / 2} C _ {0} ^ {1 / 2} - C _ {t} ^ {1 / 2} \right\| _ {F} ^ {2} + 3 \left\| C _ {t} ^ {1 / 2} C _ {0} ^ {- 1 / 2} \left(\boldsymbol {m} _ {0} - \boldsymbol {\mu} _ {0}\right) \right\| ^ {2} + 3 \| \boldsymbol {\mu} _ {t} - \boldsymbol {m} _ {t} \| ^ {2} \\ =: I _ {5} + I _ {6} + I _ {7}. \\ \end{array}
$$
$$
I _ {6} \leq 3 \| C _ {t} \| _ {F} \| C _ {0} ^ {- 1} \| _ {F} \| \boldsymbol {m} _ {0} - \boldsymbol {\mu} _ {0} \| ^ {2},
$$
$$
I _ {7} \leq 3 F _ {7} \left(\| C _ {0} - \Sigma_ {0} \| _ {F} ^ {2} + \| \boldsymbol {m} _ {0} - \boldsymbol {\mu} _ {0} \| ^ {2}\right).
$$
By the central limit theorem  $\sqrt{N} (\pmb{m}_0 - \pmb{\mu}_0)$  converges to  $\mathcal{N}(\mathbf{0},\Sigma_0)$  in distribution. Thus,  $\sqrt{\frac{N}{\log\log N}} (\pmb{m}_0 - \pmb{\mu}_0)$  converges to  $\mathbf{0}$  in distribution and hence also in probability. (There could even be almost sure results using the law of iterated logarithm but converge in probability is good enough.)
Similarly by CLT every entries of  $\sqrt{N} (C_0 - \Sigma_0)$  converges to a Gaussian distribution. Thus,  $\sqrt{\frac{N}{\log\log N}} (C_0 - \Sigma_0)$  converges to zero matrix in probability. Therefore, we have  $\frac{N}{\log\log N}\| C_t - \Sigma_t\| _F^2\to 0$  in probability.
By Step II, we have  $\| C_t\| _F\leq \| Q\| _F + F_3$ . All these constants here  $(F_{3},F_{7},\| C_{0}^{-1}\|_{F})$  can be seen or chosen as a continuous function of  $Q,b,C_0,\pmb {m}_0,\Sigma_0,\pmb {\mu}_0$  and by continuous mapping theorem they converge to the values of the same function with  $C_0 = \Sigma_0$  and  $m_0 = \mu_0$ . Thus, we conclude that  $\frac{N}{\log\log N} (I_6 + I_7)\to 0$  in probability.
Now we derive that
$$
\begin{array}{l} I _ {5} \leq \left\| \left(\Sigma_ {t} ^ {1 / 2} - C _ {t} ^ {1 / 2}\right) \Sigma_ {0} ^ {- 1 / 2} C _ {0} ^ {1 / 2} + C _ {t} ^ {1 / 2} \Sigma_ {0} ^ {- 1 / 2} \left(C _ {0} ^ {1 / 2} - \Sigma_ {0} ^ {1 / 2}\right) \right\| _ {F} ^ {2} \\ \leq 2 \| \Sigma_ {t} ^ {1 / 2} - C _ {t} ^ {1 / 2} \| _ {F} ^ {2} \| \Sigma_ {0} ^ {- 1} \| _ {F} \| C _ {0} \| _ {F} + \| C _ {t} \| _ {F} \| \Sigma_ {0} ^ {- 1} \| _ {F} \| C _ {0} ^ {1 / 2} - \Sigma_ {0} ^ {1 / 2} \| _ {F} ^ {2}. \\ \end{array}
$$
Now we show a lemma: Suppose  $A, B \in \operatorname{Sym}^{+}(d, \mathbb{R})$  are two positive definite matrices. Then we have  $\| A^{1/2} - B^{1/2} \| \leq \frac{1}{2\sqrt{\lambda}} \| A - B \|$  where  $\lambda$  is the smallest eigenvalue of  $A$  and  $B$ . Note that we are using the spectral norm here.
In fact, denote the largest eigenvector of  $A^{1/2} - B^{1/2}$  by  $\eta$ , and let  $\pmb{x} \in \mathbb{R}^d$  be the corresponding eigenvector such that  $\pmb{x}^\top \pmb{x} = 1$ . We have
$$
\begin{array}{l} \left\| A - B \right\| \geq \boldsymbol {x} ^ {\top} (A - B) \boldsymbol {x} \\ = \boldsymbol {x} ^ {\top} A ^ {1 / 2} \left(A ^ {1 / 2} - B ^ {1 / 2}\right) \boldsymbol {x} + \boldsymbol {x} ^ {\top} \left(A ^ {1 / 2} - B ^ {1 / 2}\right) B ^ {1 / 2} \boldsymbol {x} \\ = \eta \boldsymbol {x} ^ {\top} \left(A ^ {1 / 2} + B ^ {1 / 2}\right) \boldsymbol {x} \geq 2 \eta \sqrt {\lambda}. \\ \end{array}
$$
Thus, we have that  $\| A^{1 / 2} - B^{1 / 2}\| = \eta \leq \frac{1}{2\sqrt{\lambda}}\| A - B\|$ . Moreover, the Frobenius norm is bounded by
$$
\| A ^ {1 / 2} - B ^ {1 / 2} \| _ {F} \leq \sqrt {d} \| A ^ {1 / 2} - B ^ {1 / 2} \| \leq \frac {\sqrt {d}}{2 \sqrt {\lambda}} \| A - B \| \leq \frac {\sqrt {d}}{2 \sqrt {\lambda}} \| A - B \| _ {F}.
$$
Applying this lemma, we know
$$
\left\| C _ {0} ^ {1 / 2} - \Sigma_ {0} ^ {1 / 2} \right\| _ {F} \leq \frac {\sqrt {d}}{2 \sqrt {\lambda}} \| \Sigma_ {0} - C _ {0} \| _ {F},
$$
where  $\lambda$  is the smallest eigenvalue of  $C_0$  and  $\Sigma_0$ , which converges to the smallest eigenvalue of  $\Sigma_0$  as  $N$  goes to infinity.
Next we need to show that the smallest eigenvalue of  $C_t$  and  $\Sigma_t$  are uniformly (in time) lower bounded by some  $\lambda' > 0$  (depending on  $Q, b, C_0, m_0, \Sigma_0, \mu_0$ ). We revisit Step II, where we show that  $E(m_t, C_t) \leq E(m_0, C_t)$ . Then
$$
\operatorname {t r} \left(Q ^ {- 1} C _ {t} - I\right) - \log \det  \left(Q ^ {- 1} C _ {t}\right) \leq 2 E \left(\boldsymbol {m} _ {0}, C _ {0}\right)
$$
leads to a uniform lower bound of the smallest eigenvalue of  $C_t$  since  $\operatorname{tr}(Q^{-1}C_t - I) - \log \det(Q^{-1}C_t) \to \infty$  as the smallest eigenvalue of  $C_t$  goes down to zero. Thus, we have
$$
\left\| C _ {t} ^ {1 / 2} - \Sigma_ {t} ^ {1 / 2} \right\| _ {F} ^ {2} \leq \frac {d}{4 \lambda^ {\prime}} \| \Sigma_ {t} - C _ {t} \| _ {F} ^ {2} \leq \frac {d F _ {7}}{4 \lambda^ {\prime}} \left(\| C _ {0} - \Sigma_ {0} \| _ {F} ^ {2} + \| \boldsymbol {m} _ {0} - \boldsymbol {\mu} _ {0} \| ^ {2}\right).
$$
Following similar arguments we conclude that  $\frac{N}{\log\log N} I_5\to 0$  in probability as  $N\rightarrow \infty$ . Therefore, in probability
$$
\frac {N}{\log \log N} \frac {1}{N} \sum_ {i = 1} ^ {N} \left\| \widetilde {\boldsymbol {x}} _ {i} ^ {(t)} - \boldsymbol {y} _ {i} ^ {(t)} \right\| \leq \frac {N}{\log \log N} (I _ {5} + I _ {6} + I _ {7}) \to 0,
$$
$$
\mathbb {E} \left[ \mathcal {W} _ {2} ^ {2} \left(\xi_ {N} ^ {(t)}, \tilde {\zeta} _ {N} ^ {(t)}\right) \right] = o \left(\frac {\log \log N}{N}\right).
$$
Step VI. Apply Lemma L.1 to get the final result. Note that  $\boldsymbol{x}_i^{(t)}$  are i.i.d. from  $\mathcal{N}(\mathbf{0}, I)$  and  $\boldsymbol{y}_i^{(t)}$  is a linear function of  $\boldsymbol{x}_t^{(t)}$  (unlike  $\boldsymbol{m}_t$  and  $C_t$  which are random,  $\boldsymbol{\mu}_t$  and  $\Sigma_t$  are deterministic). By Lemma L.1 and Step II ( $\boldsymbol{\mu}_t$  and  $\Sigma_t$  are uniformly bounded) we have
$$
\mathbb {E} \left[ \mathcal {W} _ {2} ^ {2} \left(\xi_ {N} ^ {(t)}, \rho_ {t}\right) \right] \leq C _ {Q, \boldsymbol {b}, \Sigma_ {0}, \boldsymbol {\mu} _ {0}} \times \left\{ \begin{array}{c l} N ^ {- 1} \log \log N & \text {i f} d = 1 \\ N ^ {- 1} (\log N) ^ {2} & \text {i f} d = 2 \\ N ^ {- 2 / d} & \text {i f} d \geq 3 \end{array} . \right. \tag {87}
$$
$$
\begin{array}{l} \mathbb {E} \left[ \mathcal {W} _ {2} ^ {2} \left(\zeta_ {N} ^ {(t)}, \rho_ {t}\right) \right] \stackrel {(*)} {=} \mathbb {E} \left[ \mathcal {W} _ {2} ^ {2} \left(\widetilde {\zeta} _ {N} ^ {(t)}, \rho_ {t}\right) \right] \\ \stackrel {(*)} {\leq} \mathbb {E} \left[ \left(\mathcal {W} _ {2} (\widetilde {\zeta} _ {N} ^ {(t)}, \xi_ {N} ^ {(t)}) + \mathcal {W} _ {2} (\xi_ {N} ^ {(t)}, \rho_ {t})\right) ^ {2} \right] \\ \leq 2 \mathbb {E} \left[ \mathcal {W} _ {2} ^ {2} \left(\xi_ {N} ^ {(t)}, \widetilde {\zeta} _ {N} ^ {(t)}\right) \right] + 2 \mathbb {E} \left[ \mathcal {W} _ {2} ^ {2} \left(\xi_ {N} ^ {(t)}, \rho_ {t}\right) \right] \\ \stackrel {(* * *)} {\leq} C _ {Q, \boldsymbol {b}, \Sigma_ {0}, \boldsymbol {\mu} _ {0}} \times \left\{ \begin{array}{c l} N ^ {- 1} \log \log N & \text {i f} d = 1 \\ N ^ {- 1} (\log N) ^ {2} & \text {i f} d = 2 \\ N ^ {- 2 / d} & \text {i f} d \geq 3 \end{array} \right.. \\ \end{array}
$$
Note that we have used Step I in  $(\ast)$ , the triangle inequality in  $(\ast \ast)$ , and Step V along with (87) in  $(\ast \ast \ast)$ .
Proof of Theorem F.4. The proof is roughly similar to that of Theorem 3.7. But Step III is a little different (can be strengthened and simplified at the same time).
We show the global contraction of  $\| \pmb{m}_t - \pmb{\mu}_t\|$  and also the contraction of  $\| C_t - \Sigma_t\| _F$  after sufficient time. Note here  $\| \cdot \| _F$  is the Frobenius norm.
Frist we check the derivative of squared Euclidean norm of  $m_t - \mu_t$ .
$$
\begin{array}{l} \frac {1}{2} \frac {\mathrm {d}}{\mathrm {d} t} \| \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} \| ^ {2} = (\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}) ^ {\top} (\dot {\boldsymbol {m}} _ {t} - \dot {\boldsymbol {\mu}} _ {t}) \\ = - \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) ^ {\top} Q ^ {- 1} \left(\boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t}\right) \leq - \frac {1}{q _ {\max }} \| \boldsymbol {m} _ {t} - \boldsymbol {\mu} _ {t} \| ^ {2}. \\ \end{array}
$$
Thus,  $\| \pmb{m}_t - \pmb{\mu}_t\| \leq e^{-\frac{t}{q_{\mathrm{max}}}}\| \pmb{m}_0 - \pmb{\mu}_0\|$
To bound  $\| C_t - \Sigma_t\| _F$  we define
$$
\mathcal {S} _ {\epsilon} := \left\{S \in \operatorname {S y m} ^ {+} (d, \mathbb {R}): (1 - \epsilon) Q \preceq S \preceq (1 + \epsilon) Q \right\}.
$$
This time we do not need the relative energy but can directly check the derivative of the squared Frobenius norm:
$$
\begin{array}{l} \frac {1}{2} \frac {\mathrm {d}}{\mathrm {d} t} \| C _ {t} - \Sigma_ {t} \| _ {F} ^ {2} = \frac {1}{2} \frac {\mathrm {d}}{\mathrm {d} t} \operatorname {t r} \left(\left(C _ {t} - \Sigma_ {t}\right) ^ {2}\right) \stackrel {\operatorname {t r}} {=} \left(C _ {t} - \Sigma_ {t}\right) \left(\dot {C} _ {t} - \dot {\Sigma} _ {t}\right) \\ \stackrel {\text {t r}} {=} 2 \left(C _ {t} - \Sigma_ {t}\right) ^ {2} - 2 \left(C _ {t} - \Sigma_ {t}\right) \left(C _ {t} ^ {2} - \Sigma_ {t} ^ {2}\right) Q ^ {- 1} \\ \stackrel {\mathrm {t r}} {=} 2 \left(C _ {t} - \Sigma_ {t}\right) ^ {2} - \left(C _ {t} - \Sigma_ {t}\right) \left(C _ {t} + \Sigma_ {t}\right) \left(C _ {t} - \Sigma_ {t}\right) Q ^ {- 1} - \left(C _ {t} - \Sigma_ {t}\right) ^ {2} \left(C _ {t} + \Sigma_ {t}\right) Q ^ {- 1} \\ \stackrel {\text {t r}} {=} 2 \left(C _ {t} - \Sigma_ {t}\right) ^ {2} - \left(C _ {t} - \Sigma_ {t}\right) ^ {2} \left(C _ {t} + \Sigma_ {t}\right) Q ^ {- 1} - \left(C _ {t} - \Sigma_ {t}\right) \left(C _ {t} + \Sigma_ {t}\right) \left(C _ {t} - \Sigma_ {t}\right) Q ^ {- 1}. \tag {88} \\ \end{array}
$$