SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \widehat {\mathbf {C} - \mathbf {C} ^ {\top}} = - \mathbf {C} ^ {2} - 2 \gamma \mathbf {C} (I _ {k} - \boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}) ^ {- 1} - 2 \gamma (I _ {k} - \boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}) ^ {- 1} \mathbf {C} + \mathbf {C} + (\mathbf {C} ^ {\top}) ^ {2} \\ + 2 \gamma (I _ {k} - \boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}) ^ {- 1} \mathbf {C} ^ {\top} + 2 \gamma \mathbf {C} ^ {\top} (I _ {k} - \boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}) ^ {- 1} - \mathbf {C} ^ {\top} \\ = - \left[ \mathbf {C} ^ {2} - \left(\mathbf {C} ^ {\top}\right) ^ {2} \right] + \left(\mathbf {C} - \mathbf {C} ^ {\top}\right) - 2 \gamma \left(I _ {k} - \boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}\right) ^ {- 1} \left(\mathbf {C} - \mathbf {C} ^ {\top}\right) - 2 \gamma \left(\mathbf {C} - \mathbf {C} ^ {\top}\right) \left(I _ {k} - \boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}\right) ^ {- 1} \\ = \left[ \frac {1}{2} \left(\mathbf {I} - \mathbf {C} - \mathbf {C} ^ {\top}\right) - 2 \gamma \left(I _ {k} - \boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}\right) ^ {- 1} \right] (\mathbf {C} - \mathbf {C} ^ {\top}) \\ - \left(\mathbf {C} - \mathbf {C} ^ {\top}\right) \left[ \frac {1}{2} \left(\mathbf {I} - \mathbf {C} - \mathbf {C} ^ {\top}\right) - 2 \gamma \left(I _ {k} - \boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}\right) ^ {- 1} \right]. \\ \end{array}
$$
Finally, since  $\mathbf{C}(0) - \mathbf{C}^{\top}(0) = 0$ , it is a fixed point of the above equation and it implies that  $\mathbf{C} = \mathbf{C}^{\top}, \forall t$ . Thus, identity (C.17) is proven.
We proceed similarly for identity (C.18).
$$
\widehat {\dot {\mathbf {R} \alpha^ {\top}}} = - 2 \gamma \mathbf {R} \boldsymbol {\alpha} ^ {\top} (I _ {d} - \boldsymbol {\alpha} \boldsymbol {\alpha} ^ {\top}) ^ {- 1} - 2 \gamma (I _ {d} - \boldsymbol {\alpha} \boldsymbol {\alpha} ^ {\top}) ^ {- 1} \mathbf {R} \boldsymbol {\alpha} ^ {\top} + \mathbf {R} \mathbf {R} ^ {\top} + \mathbf {R} \boldsymbol {\alpha} ^ {\top} - (\mathbf {R} \boldsymbol {\alpha} ^ {\top}) ^ {2}.
$$
Denoting  $\mathbf{B} := \mathbf{R}\alpha^{\top}$  and computing  $\dot{\mathbf{B}} - \dot{\mathbf{B}}^{\top}$  we get:
$$
\begin{array}{l} \widehat {\mathbf {B} - \mathbf {B} ^ {\top}} = \left[ \frac {1}{2} \left(I _ {d} - \mathbf {B} - \mathbf {B} ^ {\top}\right) - 2 \gamma \left(I _ {d} - \boldsymbol {\alpha} \boldsymbol {\alpha} ^ {\top}\right) ^ {- 1} \right] (\mathbf {B} - \mathbf {B} ^ {\top}) \\ + \left(\mathbf {B} - \mathbf {B} ^ {\top}\right) \left[ \frac {1}{2} \left(I _ {d} - \mathbf {B} - \mathbf {B} ^ {\top}\right) - 2 \gamma (I _ {d} - \boldsymbol {\alpha} \boldsymbol {\alpha} ^ {\top}) ^ {- 1} \right]. \\ \end{array}
$$
Since  $(\mathbf{B}(0) - \mathbf{B}^{\top}(0)) = 0$  is a fixed point of the above equation, identity (C.18) is proven.
Finally, we show  $\mathbf{U}_{\mathbf{R}} = U_{\beta^{*}}$  and  $\mathbf{V}_{\mathbf{R}} = V_{\beta^{*}}$  using yet another invariance, namely
$$
\mathbf {Z} _ {1} ^ {\top} \mathbf {R} \mathbf {Z} _ {2} ^ {\top} = \mathbf {Z} _ {2} \mathbf {R} ^ {\top} \mathbf {Z} _ {1}, \forall t > 0. \tag {C.22}
$$
We proceed by computing the associated time derivatives and showing that their difference is null:
$$
\begin{array}{l} \widehat {\mathbf {Z} _ {1} ^ {\top} \mathbf {R} \mathbf {Z} _ {2} ^ {\top}} - \widehat {\mathbf {Z} _ {2} \mathbf {R} ^ {\top} \mathbf {Z} _ {1}} = \mathbf {Z} _ {1} ^ {\top} \dot {\mathbf {R}} \mathbf {Z} _ {2} ^ {\top} - \mathbf {Z} _ {2} \dot {\mathbf {R}} \mathbf {Z} _ {1} (C.23) \\ = - \mathbf {Z} _ {1} ^ {\top} \mathbf {R} \mathbf {Z} _ {2} ^ {\top} \mathbf {Z} _ {2} \mathbf {Z} _ {2} ^ {\top} - 2 \gamma \mathbf {Z} _ {1} ^ {\top} \mathbf {R} \mathbf {Z} _ {2} ^ {\top} - \mathbf {Z} _ {2} \mathbf {Z} _ {2} ^ {\top} \mathbf {Z} _ {1} ^ {\top} \mathbf {R} \mathbf {Z} _ {2} ^ {\top} (C.24) \\ + \mathbf {Z} _ {2} \mathbf {Z} _ {2} ^ {\top} \mathbf {Z} _ {2} \mathbf {R} ^ {\top} \mathbf {Z} _ {1} + 2 \gamma \mathbf {Z} _ {2} \mathbf {R} ^ {\top} \mathbf {Z} _ {1} + \mathbf {Z} _ {2} \mathbf {R} ^ {\top} \mathbf {Z} _ {1} \mathbf {Z} _ {2} \mathbf {Z} _ {2} ^ {\top} (C.25) \\ = \left(\mathbf {Z} _ {2} \mathbf {R} ^ {\top} \mathbf {Z} _ {1} - \mathbf {Z} _ {1} ^ {\top} \mathbf {R} \mathbf {Z} _ {2} ^ {\top}\right) \mathbf {Z} _ {2} \mathbf {Z} _ {2} ^ {\top} + \mathbf {Z} _ {2} \mathbf {Z} _ {2} ^ {\top} \left(\mathbf {Z} _ {2} \mathbf {R} ^ {\top} \mathbf {Z} _ {1} - \mathbf {Z} _ {1} ^ {\top} \mathbf {R} \mathbf {Z} _ {2} ^ {\top}\right) (C.26) \\ - 2 \gamma \left(\mathbf {Z} _ {1} ^ {\top} \mathbf {R} \mathbf {Z} _ {2} ^ {\top} - \mathbf {Z} _ {2} \mathbf {R} ^ {\top} \mathbf {Z} _ {1}\right), (C.27) \\ \end{array}
$$
where we used (C.20) and the invariance (C.8). Since  $\mathbf{Z}_2(0)\mathbf{R}^\top (0)\mathbf{Z}_1(0) - \mathbf{Z}_1(0)^\top \mathbf{R}(0)\mathbf{Z}_2(0)^\top = 0$ , it is a fixed point of the above equation and we have shown (C.22).
Finally, it holds that
$$
\begin{array}{l} \mathbf {Z} _ {1} ^ {\top} R \mathbf {Z} _ {2} ^ {\top} = \mathbf {Z} _ {2} \mathbf {R} ^ {\top} \mathbf {Z} _ {1} \iff \mathbf {Z} _ {1} ^ {\top} \beta^ {*} \mathbf {Z} _ {2} ^ {\top} - \mathbf {Z} _ {1} ^ {\top} \beta \mathbf {Z} _ {2} ^ {\top} = \mathbf {Z} _ {2} \beta^ {* \top} \mathbf {Z} _ {1} - \mathbf {Z} _ {2} \beta^ {\top} \mathbf {Z} _ {1} \\ \Longleftrightarrow \mathbf {Z} _ {1} ^ {\top} \boldsymbol {\beta} ^ {*} \mathbf {Z} _ {2} ^ {\top} = \mathbf {Z} _ {2} \boldsymbol {\beta} ^ {* \top} \mathbf {Z} _ {1} \\ \Longleftrightarrow \boldsymbol {\beta} ^ {*} \boldsymbol {\alpha} ^ {\top} = \boldsymbol {\alpha} \boldsymbol {\beta} ^ {* \top}, \tag {C.28} \\ \end{array}
$$
where the second equivalence comes from the fact that  $\mathbf{Z}_1^\top \mathbf{Z}_1$  and  $\mathbf{Z}_2\mathbf{Z}_2^\top$  are simultaneously diagonalizable and thus commute (from invariance (C.8)), and the second equivalence comes from multiplying left and right with  $\mathbf{Z}_1^{-\top}$  and  $\mathbf{Z}_1^{-1}$ , respectively and the definition of  $\alpha$ . Furthermore,
$$
\begin{array}{l} R ^ {\top} \boldsymbol {\alpha} = \boldsymbol {\alpha} ^ {\top} R \Longleftrightarrow \boldsymbol {\beta} ^ {* ^ {\top}} \boldsymbol {\alpha} - \beta^ {\top} \boldsymbol {\alpha} = \boldsymbol {\alpha} ^ {\top} \boldsymbol {\beta} ^ {*} - \boldsymbol {\alpha} ^ {\top} \boldsymbol {\beta} \\ \Longleftrightarrow \boldsymbol {\beta} ^ {* ^ {\top}} \boldsymbol {\alpha} = \boldsymbol {\alpha} ^ {\top} \boldsymbol {\beta} ^ {*}, \tag {C.29} \\ \end{array}
$$
where the second line comes from the fact that  $\beta^{\top}\alpha = \alpha^{\top}\mathbf{Z}_{1}\mathbf{Z}_{1}^{\top}\alpha = \alpha^{\top}\beta$
From repeated applications of (C.28) and (C.29) we obtain that
$$
\left(\boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}\right) \left(\boldsymbol {\beta} ^ {* \top} \boldsymbol {\beta} ^ {*}\right) = \left(\boldsymbol {\beta} ^ {* \top} \boldsymbol {\beta} ^ {*}\right) \left(\boldsymbol {\alpha} ^ {\top} \boldsymbol {\alpha}\right) \quad \text {a n d} \quad \left(\boldsymbol {\alpha} \boldsymbol {\alpha} ^ {\top}\right) \left(\boldsymbol {\beta} ^ {*} \boldsymbol {\beta} ^ {* \top}\right) = \left(\boldsymbol {\beta} ^ {*} \boldsymbol {\beta} ^ {* \top}\right) \left(\boldsymbol {\alpha} \boldsymbol {\alpha} ^ {\top}\right), \tag {C.30}
$$
and the final identity follows.
Theorem C.12 (Time evolution of singular values - orthogonal data). Assume that  $X^{\top}X = I_{d}$ . Then, the  $i^{th}$  singular value of  $\beta$  evolves over time as
$$
\sigma_ {\beta , i} (t) = \frac {2 \gamma \left(\sqrt {1 + \frac {\gamma^ {2}}{\sigma_ {\star , i} ^ {2}}} \left[ \frac {\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \gamma)}{\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} + \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \gamma)} \right] - \frac {\gamma}{\sigma_ {\star , i}}\right)}{1 - \left(\sqrt {1 + \frac {\gamma^ {2}}{\sigma_ {\star , i} ^ {2}}} \left[ \frac {\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2}+ \gamma^ {2}} - \gamma)}{\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} + \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \gamma)} \right] - \frac {\gamma}{\sigma {} _ {\star , i}}\right) ^ {2}}.
$$
As a consequence, under vanishing initialization  $\gamma \rightarrow 0$  and with a rescaling of time as  $t\to \ln (1 / \gamma)t$ , the  $i^{th}$  singular value is learned at time  $T_{i} = 1 / 2\sigma_{\star ,i}$ :
$$
\lim_{\gamma \to 0}\sigma_{\boldsymbol{\beta},i}(\ln (1 / \gamma)t) = \left\{ \begin{array}{l}0,if t <   1 / 2\sigma_{\star ,i}\\ \sigma_{\star ,i},otherwise. \end{array} \right.
$$
Proof. From Lemma 3.3 we have that  $\dot{\pmb{\alpha}} = \mathbf{R} - \alpha \mathbf{R}\pmb{\alpha}^{\top}$ . In light of Theorem C.11, identity (C.18) and it holds that:
$$
\dot {\mathbf {D}} _ {\boldsymbol {\alpha}} = \left(I _ {d} - \tilde {\mathbf {D}} _ {\boldsymbol {\alpha}} ^ {2}\right) \mathbf {D} _ {\boldsymbol {\beta} ^ {*}} - 2 \gamma \mathbf {D} _ {\boldsymbol {\alpha}}.
$$
For the  $i^{th}$  singular value of  $\alpha$  it holds that:
$$
\begin{array}{l} \widehat {\sigma_ {\boldsymbol {\alpha} , i} (t)} = - \sigma_ {\star , i} \sigma_ {\boldsymbol {\alpha}, i} ^ {2} (t) - 2 \gamma \sigma_ {\boldsymbol {\alpha}, i} (t) + \sigma_ {\star , i} \\ = - \sigma_ {\star , i} \left[ \left(\sigma_ {\alpha , i} (t) + \frac {\gamma}{\sigma_ {\star , i}}\right) ^ {2} - \left(1 + \frac {\gamma^ {2}}{\sigma_ {\star , i} ^ {2}}\right) \right], \tag {C.31} \\ \end{array}
$$
where  $\sigma_{\star,i}$  is the  $i^{th}$  singular value of  $\beta^*$ . Equation (C.31) is a Ricatti ODE and is separable. For ease of notation, let  $p := \frac{\gamma}{\sigma_{\star,i}}$ ,  $q := 1 + \frac{\gamma^2}{\sigma_{\star,i}^2}$  and  $r := -\sigma_{\star,i}$ . Then, we need to solve the IVP:
$$
\left\{ \begin{array}{l l} \widehat {\sigma_ {\alpha , i} (t)} & = r \left[ \left(\sigma_ {\alpha , i} (t) + p\right) ^ {2} - q \right] \\ \sigma_ {\alpha , i} (0) & = 0, \end{array} \right. \tag {C.32}
$$
for which we get  $\sigma_{\alpha,i}(t) = \sqrt{q}\left[\frac{1 - \exp\big(2r\sqrt{q}(t + c_1)\big)}{1 + \exp\big(2r\sqrt{q}(t + c_1)\big)}\right] - p$  and solving for the initial value gives us  $c_1 = \frac{1}{2r\sqrt{q}}\log \left(\frac{\sqrt{q} - p}{\sqrt{q} + p}\right)$ . Replacing  $p, q, r$  and rearranging we finally obtain:
$$
\sigma_ {\alpha , i} (t) = \sqrt {1 + \frac {\gamma^ {2}}{\sigma_ {\star , i} ^ {2}}} \left[ \frac {\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \gamma)}{\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} + \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \gamma)} \right] - \frac {\gamma}{\sigma_ {\star , i}}. \tag {C.33}
$$
In order to obtain the dynamics for the singular values of  $\beta = \mathbf{Z}_1\mathbf{Z}_1^\top \alpha$  we recall the relation given in Lemma C.10:
$$
\tilde {\mathbf {D}} _ {\mathbf {Z} _ {1}} ^ {2} = \gamma I _ {d} + \sqrt {\gamma^ {2} I _ {d} + \tilde {\mathbf {D}} _ {\boldsymbol {\beta}} ^ {2}},
$$
where the appropriate dimensionality of the diagonal matrices is evident from the indexing of the identity matrices. Therefore, the singular values  $\sigma_{\beta ,i}(t)$  are the solutions to the equation
$$
\sigma_ {\boldsymbol {\beta}, i} (t) = \sigma_ {\mathbf {Z} _ {1}, i} ^ {2} (t) \sigma_ {\boldsymbol {\alpha}, i} (t) = \sigma_ {\boldsymbol {\alpha}, i} (t) \left[ \gamma + \sqrt {\gamma^ {2} + \sigma_ {\boldsymbol {\beta} , i} ^ {2} (t)} \right],
$$
and have the following expression:
$$
\begin{array}{l} \sigma_ {\boldsymbol {\beta}, i} (t) = \frac {2 \gamma \sigma_ {\boldsymbol {\alpha} , i} (t)}{1 - \sigma_ {\boldsymbol {\alpha} , i} ^ {2} (t)} \\ = \frac {2 \gamma \left(\sqrt {1 + \frac {\gamma^ {2}}{\sigma_ {\star , i} ^ {2}}} \left[ \frac {\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \gamma)}{\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} + \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \gamma)} \right] - \frac {\gamma}{\sigma_ {\star , i}}\right)}{1 - \left(\sqrt {1 + \frac {\gamma^ {2}}{\sigma_ {\star , i} ^ {2}}} \left[ \frac {\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2}+ \gamma^ {2}} - \gamma)}{\gamma + \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} + \exp (- 2 t \sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}}) (\sqrt {\sigma_ {\star , i} ^ {2}+ \gamma^ {2}} - \gamma)} \right] - \frac {\gamma}{\sigma_ {\star , i}}\right) ^ {2}}. \\ \end{array}
$$
Note that  $\lim_{t\to \infty}\sigma_{\pmb{\alpha},i}(t) = \frac{1}{\sigma_{\star,i}}\left(\sqrt{\sigma_{\star,i}^2 + \gamma^2} -\gamma\right)$  and therefore we can verify that  $\beta \stackrel {t\to \infty}{\longrightarrow}\beta^{*}$  by looking at the singular values, since the singular vectors are static (Lemma C.11).
$$
\begin{array}{l} \lim  _ {t \rightarrow \infty} \sigma_ {\beta , i} (t) = \frac {2 \gamma \left(\sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \gamma\right)}{\frac {2 \gamma}{\sigma_ {\star , i}} \left(\sqrt {\sigma_ {\star , i} ^ {2} + \gamma^ {2}} - \gamma\right)} \\ = \sigma_ {\star , i}. \\ \end{array}
$$
We can now derive asymptotic transition times at which the singular values are learned. Now, we further process expression (C.33). Let  $v := \sqrt{\sigma_{\star,i}^2 + \gamma^2} - \gamma$ ,  $w := \sqrt{\sigma_{\star,i}^2 + \gamma^2} + \gamma$ , then
$v + w = 2\sqrt{\sigma_{\star,i}^2 + \gamma^2}$  and  $w - v = 2\gamma$  . We re-write:
$$
\begin{array}{l} \sigma_ {\boldsymbol {\alpha}, i} (t) = \frac {1}{2 \sigma_ {\star , i}} \left[ (w + v) \left[ \frac {w - \exp (- t (v + w)) v}{w + \exp (- t (v + w)) v} \right] - (w - v) \right] \\ = \frac {1}{2 \sigma_ {\star , i}} \left[ w \left(\frac {w - \exp (- t (v + w)) v}{w + \exp (- t (v + w)) v} - 1\right) + v \left(\frac {w - \exp (- t (v + w)) v}{w + \exp (- t (v + w)) v} + 1\right) \right] \\ = \frac {1}{\sigma_ {\star , i}} \left[ \frac {- v w \exp (- t (v + w))}{w + \exp (- t (v + w)) v} + \frac {v w}{w + \exp (- t (v + w)) v} \right] \\ = \frac {v}{\sigma_ {\star , i}} \left[ 1 - \underbrace {\frac {(v + w) \exp (- t (v + w))}{w + v \exp (- t (v + w))}} _ {=: h (t)} \right]. \tag {C.34} \\ \end{array}
$$
Since from before we have that  $\sigma_{\beta,i}(t) = \frac{2\gamma\sigma_{\alpha,i}(t)}{1 - \sigma_{\alpha,i}^2(t)}$  and we can write  $1 - \sigma_{\alpha,i}^{2}(t) = \frac{2\gamma v}{\sigma_{\star,i}^{2}} + \frac{v^{2}h(t)(2 - h(t))}{\sigma_{\star,i}^{2}}$ , the singular values of  $\beta$  become:
$$
\sigma_ {\boldsymbol {\beta}, i} (t) = \frac {2 \gamma \sigma_ {\boldsymbol {\alpha} , i} (t)}{1 - \sigma_ {\boldsymbol {\alpha} , i} ^ {2} (t)} = \frac {\frac {2 v \gamma}{\sigma_ {\star , i}} [ 1 - h (t) ]}{\frac {2 \gamma v}{\sigma_ {\star , i} ^ {2}} + \frac {v ^ {2} h (t) (2 - h (t))}{\sigma_ {\star , i} ^ {2}}} = \frac {\sigma_ {\star , i} [ 1 - h (t) ]}{1 + \frac {v h (t) (2 - h (t))}{2 \gamma}}.
$$
We wish to study the limit of infinitesimal initialization  $\gamma \rightarrow 0$ . We introduce a constant  $c > 0$  and rewrite  $\gamma = \exp(-c)$  and have  $c = \ln(1 / \gamma)$ . Rescaling time  $t \to ct$  and taking the limit  $c \to \infty$  we have
$$
\begin{array}{l} \lim _ {c \to \infty} \sigma_ {\boldsymbol {\beta}, i} (c t) = \lim _ {c \to \infty} \frac {2 \sigma_ {\star , i} [ 1 - h (c t) ]}{2 + v h (c t) \exp (c) (2 - h (c t))} \\ = \frac {\sigma_ {\star , i}}{1 + \lim  _ {c \rightarrow \infty} v h (c t) \exp (c)} \\ = \left\{ \begin{array}{l} 0, \text {i f} t <   1 / 2 \sigma_ {\star , i}, \\ \frac {\sigma_ {\star , i}}{1 + 2 \sigma_ {\star , i}}, \text {i f} t = 1 / 2 \sigma_ {\star , i} \\ \sigma_ {\star , i}, \text {i f} t > 1 / 2 \sigma_ {\star , i}, \end{array} \right. \\ \end{array}
$$
since  $\lim_{c\to \infty}\exp (c)h(ct) = \lim_{c\to \infty}\frac{(v + w)\exp(c[1 - t(v + w)])}{w + v\exp(-ct(v + w))}$
$$
\begin{array}{l} = \lim _ {c \to \infty} \frac {2 \sqrt {\sigma_ {\star , i} ^ {2} + \exp (- 2 c)} \exp (c [ 1 - 2 t \sigma_ {\star , i} ]) \exp \left(\frac {- 2 c t \exp (- 2 c)}{\sigma_ {\star , i} + \sqrt {\sigma_ {\star , i} ^ {2}} + \exp (- 2 c)}\right)}{\sqrt {\sigma_ {\star , i} ^ {2} + \exp (- 2 c)} + \exp (- c) + v \exp (- c t (v + w))} \\ = \left\{ \begin{array}{l} \infty , \text {i f} t <   1 / 2 \sigma_ {\star , i} \\ 2, \text {i f} t = 1 / 2 \sigma_ {\star , i} \\ 0, \text {i f} t > 1 / 2 \sigma_ {\star , i}. \end{array} \right. \\ \end{array}
$$
Lemma C.13 (Mirror on singular values). With the same notations as in Theorem 3.1, The singular values of  $\beta$ , denoted by  $\mathbf{D}_{\beta}$ , follow the mirror flow
$$
\mathrm {d} \nabla \Psi_ {\gamma} (\mathbf {D} _ {\boldsymbol {\beta}}) = - \nabla_ {\mathbf {D} _ {\boldsymbol {\beta}}} \mathcal {L} (\boldsymbol {\beta}) \mathrm {d} t,
$$
where the potential is  $\Psi_{\gamma}(\mathbf{D}_{\beta})\coloneqq \mathrm{tr}\left(\frac{1}{2}\mathbf{D}_{\beta}\sinh^{-1}(\mathbf{D}_{\beta} / \gamma) - \sqrt{\mathbf{D}_{\beta}^2 + \gamma^2}\right)$ .
Proof. We recall the dynamics induces on  $\alpha = \mathbf{Z}_1^{-\top}\mathbf{Z}_2$  given in Lemma 3.3:
$$
\dot {\boldsymbol {\alpha}} = \mathbf {R} - \boldsymbol {\alpha} \mathbf {R} ^ {\top} \boldsymbol {\alpha}. \tag {C.35}
$$
Writing the SVD decomposition of  $\alpha$  as  $\mathbf{U}_{\alpha}\mathbf{D}_{\alpha}\mathbf{V}_{\alpha}^{\top} = \mathbf{U}_{\mathbf{Z}_1}\mathbf{D}_{\alpha}\mathbf{V}_{\mathbf{Z}_2}^{\top}$  we have that:
$$
\begin{array}{l} \mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \dot {\boldsymbol {\alpha}} \mathbf {V} _ {\mathbf {Z} _ {2}} = \widehat {\mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \mathbf {U} _ {\mathbf {Z} _ {1}} \mathbf {D} _ {\boldsymbol {\alpha}} \mathbf {V} _ {\mathbf {Z} _ {2}} ^ {\top}} \mathbf {V} _ {\mathbf {Z} _ {2}} \\ = \mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \dot {\mathbf {U}} _ {\mathbf {Z} _ {1}} \mathbf {D} _ {\alpha} \mathbf {V} _ {\mathbf {Z} _ {2}} ^ {\top} \mathbf {V} _ {\mathbf {Z} _ {2}} + \mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \mathbf {U} _ {\mathbf {Z} _ {1}} \dot {\mathbf {D}} _ {\alpha} \mathbf {V} _ {\mathbf {Z} _ {2}} ^ {\top} \mathbf {V} _ {\mathbf {Z} _ {2}} + \mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \mathbf {U} _ {\mathbf {Z} _ {1}} \mathbf {D} _ {\alpha} \dot {\mathbf {V}} _ {\mathbf {Z} _ {2}} ^ {\top} \mathbf {V} _ {\mathbf {Z} _ {2}} \\ = \mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \dot {\mathbf {U}} _ {\mathbf {Z} _ {1}} \mathbf {D} _ {\boldsymbol {\alpha}} + \dot {\mathbf {D}} _ {\boldsymbol {\alpha}} + \mathbf {D} _ {\boldsymbol {\alpha}} \dot {\mathbf {V}} _ {\mathbf {Z} _ {2}} ^ {\top} \mathbf {V} _ {\mathbf {Z} _ {2}}. \\ \end{array}
$$
Since  $\mathbf{U}_{\mathbf{Z}_1}^{\top}\dot{\mathbf{U}}_{\mathbf{Z}_1} + \dot{\mathbf{U}}_{\mathbf{Z}_1}^{\top}\mathbf{U}_{\mathbf{Z}_1} = 0$  and  $\mathbf{V}_{\mathbf{Z}_2}^{\top}\dot{\mathbf{V}}_{\mathbf{Z}_2} + \dot{\mathbf{V}}_{\mathbf{Z}_2}^{\top}\mathbf{V}_{\mathbf{Z}_2} = 0$ , the matrices  $\mathbf{U}_{\mathbf{Z}_1}^{\top}\dot{\mathbf{U}}_{\mathbf{Z}_1}$  and  $\mathbf{V}_{\mathbf{Z}_2}^{\top}\dot{\mathbf{V}}_{\mathbf{Z}_2}$  are skew-symmetric (have 0 diagonal), and therefore the principal diagonals of the products  $\mathbf{U}_{\mathbf{Z}_1}^{\top}\dot{\mathbf{U}}_{\mathbf{Z}_1}\mathbf{D}_{\alpha}$  and  $\mathbf{D}_{\alpha}\dot{\mathbf{V}}_{\mathbf{Z}_2}^{\top}\mathbf{V}_{\mathbf{Z}_2}$  are also 0.
We define the linear operator  $\mathrm{diag}:\mathbb{R}^{d\times k}\to \mathbb{R}^{d\times k}$  which maps any  $A\in \mathbb{R}^{d\times k}$  to a matrix of the same dimensions whose principal diagonal contains the elements on the principal diagonal of  $A$ , and zeros otherwise. With this notation, we have that  $\mathrm{diag}(\mathbf{U}_{\mathbf{Z}_1}^\top \dot{\pmb{\alpha}}\mathbf{V}_{\mathbf{Z}_2}) = \dot{\mathbf{D}}_{\pmb{\alpha}}$
We similarly apply the orthogonal matrices to the left and right of the RHS of C.35, letting  $\mathbf{R}' \coloneqq \mathbf{U}_{\mathbf{Z}_1}^\top \mathbf{R}\mathbf{V}_{\mathbf{Z}_2} \in \mathbb{R}^{d \times k}$ :
$$
\begin{array}{l} \operatorname {d i a g} \left(\mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \left(\mathbf {R} - \alpha \mathbf {R} ^ {\top} \boldsymbol {\alpha}\right) \mathbf {V} _ {\mathbf {Z} _ {2}}\right) = \operatorname {d i a g} \left(\mathbf {R} ^ {\prime} - \mathbf {D} _ {\boldsymbol {\alpha}} \mathbf {R} ^ {\prime \top} \mathbf {D} _ {\boldsymbol {\alpha}}\right) \\ = \operatorname {d i a g} \left(\mathbf {R} ^ {\prime}\right) - \operatorname {d i a g} \left(\mathbf {D} _ {\alpha} \mathbf {R} ^ {\prime} ^ {\top} \mathbf {D} _ {\alpha}\right) \\ = \operatorname {d i a g} \left(\mathbf {R} ^ {\prime}\right) - \mathbf {D} _ {\alpha} \operatorname {d i a g} \left(\mathbf {R} ^ {\prime}\right) ^ {\top} \mathbf {D} _ {\alpha} \\ = \left(I _ {d} - \tilde {\mathbf {D}} _ {\boldsymbol {\alpha}} ^ {2}\right) \operatorname {d i a g} \left(\mathbf {R} ^ {\prime}\right). \\ \end{array}
$$
Therefore it holds that:
$$
\dot {\mathbf {D}} _ {\alpha} = \left(I _ {d} - \tilde {\mathbf {D}} _ {\alpha} ^ {2}\right) \operatorname {d i a g} \left(\mathbf {R} ^ {\prime}\right) \tag {C.36}
$$
We now wish to arrive at an expression in  $\beta$ . From the definition of  $\alpha$  it holds that  $\alpha = (\mathbf{Z}_1\mathbf{Z}_1^\top)^{-1}\beta$  and, in conjunction with the first part of Theorem C.11 it holds that  $\mathbf{D}_{\alpha} = (\tilde{\mathbf{D}}_{\mathbf{Z}_1}^2)^{-1}\mathbf{D}_{\beta}$ . Therefore, the LHS of (C.36) becomes:
$$
\dot {\mathbf {D}} _ {\alpha} = \left(\tilde {\mathbf {D}} _ {\mathbf {Z} _ {1}} ^ {2} \dot {\cdot}\right) ^ {- 1} \mathbf {D} _ {\beta} + \left(\tilde {\mathbf {D}} _ {\mathbf {Z} _ {1}} ^ {2}\right) ^ {- 1} \dot {\mathbf {D}} _ {\beta}. \tag {C.37}
$$
For computing  $(\tilde{\mathbf{D}}_{\mathbf{Z}_1}^2)^{-1}$  it is perhaps easiest to proceed as we did with  $\alpha$ .
$$
\begin{array}{l} \widehat {\mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \left(\mathbf {Z} _ {1} \mathbf {Z} _ {1} ^ {\top}\right) ^ {- 1}} \mathbf {U} _ {\mathbf {Z} _ {1}} = - \mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \left(\mathbf {Z} _ {1} \mathbf {Z} _ {1} ^ {\top}\right) ^ {- 1} \widehat {\left(\mathbf {Z} _ {1} \mathbf {Z} _ {1} ^ {\top}\right)} \left(\mathbf {Z} _ {1} \mathbf {Z} _ {1} ^ {\top}\right) ^ {- 1} \mathbf {U} _ {\mathbf {Z} _ {1}} \\ = - (\tilde {\mathbf {D}} _ {\mathbf {Z} _ {1}} ^ {2}) ^ {- 1} \left(\mathbf {D} _ {\boldsymbol {\beta}} \mathbf {R} ^ {\prime \top} + \mathbf {R} ^ {\prime} \mathbf {D} _ {\boldsymbol {\beta}} ^ {\top}\right) (\tilde {\mathbf {D}} _ {\mathbf {Z} _ {1}} ^ {2}) ^ {- 1}, \\ \end{array}
$$
$$
\begin{array}{l} \left(\tilde {\mathbf {D}} _ {\mathbf {Z} _ {1}} ^ {2 \bullet}\right) ^ {- 1} = \operatorname {d i a g} \left(\widehat {\mathbf {U} _ {\mathbf {Z} _ {1}} ^ {\top} \left(\mathbf {Z} _ {1} \mathbf {Z} _ {1} ^ {\top}\right) ^ {- 1} \mathbf {U} _ {\mathbf {Z} _ {1}}}\right) \\ = - \left(\tilde {\mathbf {D}} _ {\mathbf {Z} _ {1}} ^ {2}\right) ^ {- 1} \left(\mathbf {D} _ {\boldsymbol {\beta}} \operatorname {d i a g} \left(\mathbf {R} ^ {\prime}\right) ^ {\top} + \operatorname {d i a g} \left(\mathbf {R} ^ {\prime}\right) \mathbf {D} _ {\boldsymbol {\beta}} ^ {\top}\right) \left(\tilde {\mathbf {D}} _ {\mathbf {Z} _ {1}} ^ {2}\right) ^ {- 1} \\ = - 2 \left(\tilde {\mathbf {D}} _ {\mathbf {Z} _ {1}} ^ {2}\right) ^ {- 2} \mathbf {D} _ {\boldsymbol {\beta}} \operatorname {d i a g} \left(\mathbf {R} ^ {\prime}\right) ^ {\top}. \\ \end{array}
$$
Putting everything back together in (C.37) we have that