SlowGuess/NeurIPS_2023_OCR huggingface dataset:
Lemma 3 restated. Denote  $\tilde{\phi}^u = \arccos \left(-\frac{144\pi}{\pi + k - 1}\frac{(\mathbf{1}^\top\mathbf{a}^*)^2}{k}\log (4 + k^2)\right)$  and
$$
\varpi = \min  \left\{\sin \tilde {\phi} ^ {u} \mathbb {1} _ {\phi_ {0} \leq \tilde {\phi} ^ {u}} + \sin \phi_ {0} \mathbb {1} _ {\phi_ {0} > \tilde {\phi} ^ {u}}, \sin \left(\pi - \left((2 0 \| \boldsymbol {a} ^ {*} \| ^ {2}) ^ {- 1} \wedge 1\right)\right) \right\},
$$
$$
c \left\| \boldsymbol {a} ^ {*} \right\| ^ {2} \sqrt {\log (N d k / \delta)} \max  \left\{\eta (\sqrt {I d} + I), \sqrt {\frac {\eta}{N}} \right\} \leq \frac {\varpi^ {2}}{1 6 k}, \tag {D.25}
$$
Proof. If  $\phi_{\tau_a} \leq \tilde{\phi}^u$ , the lower bound trivially holds. Next we prove the lower bound for  $\sin \phi_{\tau_a}$  starting from the last initial region, that is  $\phi_0 \in [\tilde{\phi}^u, \pi)$ . From (D.16) and  $|\mathbf{1}^\top \mathbf{a}^*| \geq \| \mathbf{a}^*\|$ , we know that
$$
\tau_ {1, 4} = \frac {1}{\eta} \log \left(4 + \frac {1 6 (\pi - 1)}{\zeta_ {a}}\right) \leq \frac {1}{\eta} \log \left(4 + \frac {1 6 (\pi - 1)}{\frac {3 2 (\pi - 1) ^ {2} (\mathbf {1} ^ {\top} \mathbf {a} ^ {*}) ^ {2}}{k (\pi + k - 1) \| \mathbf {a} ^ {*} \| ^ {2}}}\right) \leq \frac {1}{\eta} \log (4 + k ^ {2}).
$$
Let  $C = \frac{\log(1 + 2k^2)}{4}$ , then  $e^{2C} \leq (1 + 2k^2)^{\frac{1}{2}} \leq 2k$ . Denote  $\varpi_0 = \sin \tilde{\phi}^u \mathbb{1}_{\phi_0 < \tilde{\phi}^u} + \sin \phi_0 \mathbb{1}_{\phi_0 \geq \tilde{\phi}^u}$ .
From the definition of  $\pmb{h}_t$  and (B.9), we know
$$
\left\| \boldsymbol {h} _ {t} \right\| \leq \frac {1}{N} \sum_ {i = 1} ^ {N} \left\| \mathbf {P} _ {t} ^ {i} - \mathbf {P} _ {t} \right\| \left\| \nabla_ {\boldsymbol {w}} L \left(\boldsymbol {w} _ {t} ^ {i}, \boldsymbol {a} _ {t} ^ {i}\right) \right\| \leq 5 0 c \eta (\sqrt {I d} + I) \sqrt {\log (N d / \delta)} K _ {a} ^ {4}, \tag {D.26}
$$
where we also used  $\| \nabla_{\boldsymbol{w}}L(\boldsymbol{w}_t^i,\boldsymbol{a}_t^i)\| \leq 5K_a^2$ . Recall that  $\check{\boldsymbol{v}}_t - \boldsymbol{v}_t = \eta \boldsymbol{h}_{t - 1}$ , then it follows that
$$
\begin{array}{l} \left| \left\| \tilde {\boldsymbol {v}} _ {t} \right\| ^ {2} \sin^ {2} \check {\phi} _ {t} - \left\| \boldsymbol {v} _ {t} \right\| ^ {2} \sin^ {2} \phi_ {t} \right| \\ = \left| \left\| \check {\boldsymbol {v}} _ {t} \right\| ^ {2} - \left\| \boldsymbol {v} _ {t} \right\| ^ {2} - \left(\left(\check {\boldsymbol {v}} _ {t} ^ {\top} \boldsymbol {w} ^ {*}\right) ^ {2} - \left(\boldsymbol {v} _ {t} ^ {\top} \boldsymbol {w} ^ {*}\right) ^ {2}\right) \right| \\ = \left| \left(\| \check {\boldsymbol {v}} _ {t} \| + \| \boldsymbol {v} _ {t} \|\right) \left(\| \check {\boldsymbol {v}} _ {t} \| - \| \boldsymbol {v} _ {t} \|\right) \right| + \left| \left(\left| \check {\boldsymbol {v}} _ {t} ^ {\top} \boldsymbol {w} ^ {*} \right| - \left| \boldsymbol {v} _ {t} ^ {\top} \boldsymbol {w} ^ {*} \right|\right) \left(\left| \check {\boldsymbol {v}} _ {t} ^ {\top} \boldsymbol {w} ^ {*} \right| + \left| \boldsymbol {v} _ {t} ^ {\top} \boldsymbol {w} ^ {*} \right|\right) \right| \\ \leq 7 \| \check {\boldsymbol {v}} _ {t} - \boldsymbol {v} _ {t} \| + 7 | (\check {\boldsymbol {v}} _ {t} - \boldsymbol {v} _ {t}) ^ {\top} \boldsymbol {w} ^ {*} | ^ {2} \\ \leq 1 4 \eta \| \boldsymbol {h} _ {t - 1} \|, \tag {D.27} \\ \end{array}
$$
where we used  $\| \pmb{v}_t\| \leq 3$  in Lemma 13 and  $\| \tilde{\pmb{v}}_t\| \leq \| \pmb {v}_t\| +\eta \| \pmb {h}_t\| < 4$  . For any  $t$  , we define a variable
$$
\tau_ {t} \left(K _ {a}\right) = \sup  _ {0 \leq s \leq t - 1} \left\{\min  _ {i \in [ N ]} \phi_ {s} ^ {i} \leq \pi - \left(\frac {1}{2 0 K _ {a} ^ {2}} \wedge 1\right) \right\}. \tag {D.28}
$$
If  $\tau_{t}(K_{a})$  does not exist, that is  $\min_{i\in [N]}\phi_s^i >\pi -\left((20K_a^2)^{-1}\wedge 1\right)$  holds for any  $0\leq s\leq t - 1$  we let  $\tau_t(K_a) = 0$ . Then for any  $\tau_{t}(K_{a})\leq s\leq t - 1$ , we always have  $\min_{i\in [N]}\phi_s^i >\pi -\left((20K_a^2)^{-1}\wedge 1\right)$  such that
$$
\begin{array}{l} 1 - \eta \lambda_ {s} \cos \phi_ {s} = 1 - \eta \cos \phi_ {s} \cdot \frac {1}{N} \sum_ {i = 1} ^ {N} \frac {\pi - \phi_ {s} ^ {i}}{2 \pi} \frac {\left(\boldsymbol {a} _ {s} ^ {i}\right) ^ {\top} \boldsymbol {a} ^ {*}}{\left\| \boldsymbol {v} _ {s} \right\|} \\ \geq 1 - \eta \frac {1}{N} \sum_ {i = 1} ^ {N} \frac {1}{2 \pi \cdot (2 0 K _ {a} ^ {2} \lor 1)} \frac {\left| \left(\boldsymbol {a} _ {s} ^ {i}\right) ^ {\top} \boldsymbol {a} ^ {*} \right|}{\left\| \boldsymbol {v} _ {s} \right\|} \\ \stackrel {(i i)} {\geq} 1 - \eta \frac {5 K _ {a} ^ {2}}{2 \pi \cdot (2 0 K _ {a} ^ {2} \vee 1)} \geq 1 - \frac {\eta}{8}. \tag {D.29} \\ \end{array}
$$
Without loss of generality, we assume  $\arg \min_{i\in [N]}\phi_{\tau_t(K_a)}^i = 1$ . It follows that
$$
\begin{array}{l} \sin^ {2} \phi_ {\tau_ {t} (K _ {a})} \geq \sin^ {2} \phi_ {\tau_ {t} (K _ {a})} ^ {1} - \left| \sin^ {2} \phi_ {\tau_ {t} (K _ {a})} - \sin^ {2} \phi_ {\tau_ {t} (K _ {a})} ^ {1} \right| \\ \geq \sin^ {2} \phi_ {\tau_ {t} (K _ {a})} ^ {1} - 2 \left| \cos \phi_ {\tau_ {t} (K _ {a})} - \cos \phi_ {\tau_ {t} (K _ {a})} ^ {1} \right| \\ \geq \sin^ {2} \phi_ {\tau_ {t} (K _ {a})} ^ {1} - \frac {2 \| \pmb {v} _ {\tau_ {t} (K _ {a})} ^ {1} - \pmb {v} _ {\tau_ {t} (K _ {a})} \|}{\| \pmb {v} _ {\tau_ {t} (K _ {a})} \|} - 2 \| \pmb {v} _ {\tau_ {t} (K _ {a})} ^ {1} \| \left| \frac {1}{\| \pmb {v} _ {\tau_ {t} (K _ {a})} \|} - \frac {1}{\| \pmb {v} _ {\tau_ {t} (K _ {a})} ^ {1} \|} \right| \\ \geq \sin^ {2} (\pi - ((2 0 K _ {a} ^ {2}) ^ {- 1} \wedge 1)) - 6 c \eta (\sqrt {I d} + I) \sqrt {\log (N d / \delta)} K _ {a} ^ {2} \\ \geq \frac {\sin^ {2} \left(\pi - \left(\left(2 0 K _ {a} ^ {2}\right) ^ {- 1} \wedge 1\right)\right)}{2}. \tag {D.30} \\ \end{array}
$$
Notice that if  $t \leq 4C / \eta$ , we have
$$
\left(1 - \frac {\eta}{8}\right) ^ {2 t} \geq \left(1 - \frac {\eta}{8}\right) ^ {\frac {8}{\eta} \cdot C} \geq 2 ^ {- 2 C} \geq e ^ {- 2 C} \geq \frac {1}{2 k}. \tag {D.31}
$$
It holds because  $\eta /8\leq \frac{1}{2}$  and  $f(x) = (1 - x)^{1 / x}$  is decreasing in [0, 1]. Invoking Lemma 1, we know
$$
\| \pmb {v} _ {t} \| ^ {2} \sin^ {2} \phi_ {t} = \left(1 - \eta \lambda_ {t - 1} \cos \phi_ {t - 1}\right) ^ {2} \| \pmb {v} _ {t - 1} \| ^ {2} \sin^ {2} \phi_ {t - 1} - 2 \eta M _ {1, t - 1} + \eta^ {2} M _ {2, t - 1} + H _ {t}
$$
$$
\begin{array}{l} \stackrel {(i)} {\geq} \left(1 - \frac {\eta}{8}\right) ^ {2 (t - \tau_ {t} (K _ {a}))} \| \boldsymbol {v} _ {\tau_ {t} (K _ {a})} \| ^ {2} \sin^ {2} \phi_ {\tau_ {t} (K _ {a})} - 2 \eta \sum_ {s = \tau_ {t} (K _ {a})} ^ {t - 1} \left(1 - \frac {\eta}{8}\right) ^ {2 (t - 1 - s)} (M _ {1, s} + 7 \| \boldsymbol {h} _ {s} \|) \\ \geq \left(1 - \frac {\eta}{8}\right) ^ {2 t} \frac {1}{4} \min \left\{\sin^ {2} \phi_ {\tau_ {t} (K _ {a})}, \varpi_ {0} ^ {2} \right\} - 2 \eta \sum_ {s = \tau_ {t} (K _ {a})} ^ {t - 1} \left(1 - \frac {\eta}{8}\right) ^ {2 (t - 1 - s)} (M _ {1, s} + 7 \| \pmb {h} _ {s} \|) \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(i i)} {\geq} \left(1 - \frac {\eta}{8}\right) ^ {2 t} \frac {1}{4} \min  \left\{\frac {\sin^ {2} \left[ \pi - \left(\left(2 0 K _ {a} ^ {2}\right) ^ {- 1} \wedge 1\right) \right]}{2}, \pi_ {0} ^ {2} \right\} \\ - 1 4 0 c \eta (\sqrt {I d} + I) \sqrt {\log (N d / \delta)} K _ {a} ^ {2} - 2 \eta \max  _ {0 \leq \tau \leq t - 1} \left| \sum_ {s = \tau} ^ {t - 1} \left(1 - \frac {\eta}{8}\right) ^ {2 (t - 1 - s)} M _ {1, s} \right| \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(i i i)} {\geq} \frac {1}{2 k} \cdot \frac {1}{4} \min  \left\{\frac {\sin^ {2} [ \pi - ((2 0 K _ {a} ^ {2}) ^ {- 1} \wedge 1) ]}{2}, \varpi_ {0} ^ {2} \right\} - 1 4 0 c \eta (\sqrt {I d} + I) \sqrt {\log (N d / \delta)} K _ {a} ^ {2} \\ - \frac {1 2 c K _ {a} ^ {2} \eta}{N} (3 + 5 \eta K _ {a} ^ {2}) \sqrt {\log (1 / \eta \delta) \sum_ {s = 0} ^ {t - 1} \sum_ {i = 1} ^ {N} \left(1 - \frac {\eta}{8}\right) ^ {2 (t - 1 - s)}} \\ \end{array}
$$
$$
\stackrel {(i v)} {\geq} \frac {\min \left\{\sin^ {2} \left[ \pi - \left((2 0 K _ {a} ^ {2}) ^ {- 1} \wedge 1\right) \right] , \varpi_ {0} ^ {2} \right\}}{2 5 k},
$$
where  $(i)$  holds due to (D.27) and (D.29);  $(ii)$  follows from (D.30) and (D.26);  $(iii)$  comes from (D.31) and Lemma 11; and  $(iv)$  holds due to the assumption D.25. Plugging the choice of  $C =$
$\log (1 + 2k^2) / 4$  , together with  $\| \pmb {v}_t\| \leq 3$  , we can prove the conclusion
$$
\sin^ {2} \phi_ {\tau_ {a}} \geq \frac {\min \left\{\sin^ {2} \left[ \pi - \left((2 0 K _ {a} ^ {2}) ^ {- 1} \wedge 1\right) \right] , \varpi_ {0} ^ {2} \right\}}{2 2 5 k} \geq \frac {\varpi^ {2}}{2 2 5 k}.
$$
D.3 Self-correction of the First Layer
Theorem 2 restated. For the initial point  $(\pmb{v}_0, \pmb{a}_0)$  with  $\pmb{a}_0^\top \pmb{a}^* \geq \gamma_a$  and  $\sin \phi_0 > 0$ , we denote  $\tau_v = \inf \{t \geq 0 : \phi_v \leq \tilde{\phi}^l\}$ , where  $\cos \tilde{\phi}^l = \frac{144\pi}{\pi + k - 1} \frac{(\pmb{1}^\top \pmb{a}^*)^2}{k} \log (4 + k^2)$ . If the learning rate satisfies
$$
c \| \boldsymbol {a} ^ {*} \| ^ {2} \sqrt {\log (N d k / \delta)} \max  \left\{\eta (\sqrt {I d} + I), \sqrt {\frac {\eta}{N}} \right\} \leq \sin^ {2} \phi_ {0}, \tag {D.32}
$$
Proof. Find  $\tilde{\phi}^f \in (\pi/2, \pi)$  such that  $\cos \tilde{\phi}^f = -\frac{\pi}{\pi + k - 1}\frac{(\mathbf{1}^\top\mathbf{a}^*)^2}{k}\log (4 + k^2)$ . We introduce the following stopping time w.r.t. the angle  $\phi$  during the training process,
$$
\tau_ {+} (\phi) = \inf  _ {t \geq 0} \left\{t: \phi_ {t} \leq \phi \right\} \quad \text {f o r} \quad \phi \in [ 0, \pi ].
$$
Initial angle:  $\phi_0\in (\tilde{\phi}^f,\pi)$ . For any  $s\leq \tau_{+}(\tilde{\phi}^{f})$ , under the event  $\mathcal{E}_{\mathrm{good}}$ , we have
$$
\begin{array}{l} \cos \phi_ {s} ^ {i} \leq \cos \phi_ {s} + | \cos \phi_ {s} ^ {i} - \cos \phi_ {s} | \\ \leq \cos \tilde {\phi} ^ {f} + 1 2 c \eta K _ {a} ^ {2} (\sqrt {I d} + I) \sqrt {\log (N d / \delta)} \\ \leq - \frac {\pi}{\pi + k - 1} \frac {(\mathbf {1} ^ {\top} \mathbf {a} ^ {*}) ^ {2}}{k} \log (4 + k ^ {2}) + \frac {\pi}{\pi + k - 1} \frac {(\mathbf {1} ^ {\top} \mathbf {a} ^ {*}) ^ {2}}{k} \log (4 + k ^ {2}) = 0. \\ \end{array}
$$
It means that  $\phi_s^i\geq \pi /2$  holds for any  $0\leq s\leq \min \{\tau_{+}(\tilde{\phi}^{f}),\widetilde{O} (\eta^{-2})\}$  . As a consequence, we have
$$
\frac {1}{N} \sum_ {i = 1} ^ {N} B _ {s} ^ {i} \geq \zeta_ {a} - \frac {\pi - 1}{\pi + k - 1} \frac {(\mathbf {1} ^ {\top} \mathbf {a} ^ {*}) ^ {2}}{\| \mathbf {a} ^ {*} \| ^ {2}} \left(1 - \eta \frac {\pi + k - 1}{2 \pi}\right) ^ {s}.
$$
Notice that if  $t \leq \frac{1}{\eta} \frac{2\pi}{\pi - 1}$ , we have
$$
\left(1 - \eta \frac {\pi - 1}{2 \pi}\right) ^ {t} \geq \left(1 - \eta \frac {\pi - 1}{2 \pi}\right) ^ {\frac {1}{\eta} \frac {2 \pi}{\pi - 1}} \geq \frac {1}{4},
$$
which holds because  $\eta \frac{\pi - 1}{2\pi} \leq \frac{1}{2}$  and  $(1 - x)^{1/x}$  is decreasing in  $[0, 1]$ . If  $t \geq \frac{1}{\eta} \frac{2\pi}{\pi - 1}$ , we also have
$$
1 - \left(1 - \eta \frac {\pi - 1}{2 \pi}\right) ^ {t} \geq 1 - \left(1 - \eta \frac {\pi - 1}{2 \pi}\right) ^ {\frac {1}{\eta} \frac {2 \pi}{\pi - 1}} \geq 1 - e ^ {- 1} \geq \frac {1}{4},
$$
because  $(1 - x)^{1 / x} \leq e^{-1}$  for  $x \in [0, 1]$ . Together with the dynamic (D.1) and  $\pmb{a}_0^\top \pmb{a}^* \geq \gamma_a$ , under the event  $\mathcal{E}_{\mathrm{good}}$ , we have
$$
\begin{array}{l} \pmb {a} _ {s} ^ {\top} \pmb {a} ^ {*} \geq \frac {\zeta_ {a} \| \pmb {a} ^ {*} \| ^ {2}}{2 (\pi - 1)} \left[ 1 - \left(1 - \frac {\eta (\pi - 1)}{2 \pi}\right) ^ {s} \right] - 2 c \sqrt {\frac {\eta \log (2 t / \delta)}{N}} K _ {a} ^ {2} \\ + \left(\gamma_ {a} - \frac {(\pi - 1) \left(\mathbf {1} ^ {\top} \mathbf {a} ^ {*}\right) ^ {2}}{k (\pi + k - 1)}\right) \left(1 - \frac {\eta (\pi - 1)}{2 \pi}\right) ^ {s} \\ \stackrel {(i)} {\geq} \frac {1}{4} \frac {3 2 (\pi - 1)}{k (\pi + k - 1)} + \frac {1}{4} \left(\gamma_ {a} - \frac {(\pi - 1) (\mathbf {1} ^ {\top} \mathbf {a} ^ {*}) ^ {2}}{k (\pi + k - 1)}\right) - 3 c \sqrt {\frac {\eta k \log (1 / \eta \delta)}{N}} K _ {a} ^ {2} \\ \end{array}
$$
$$
\stackrel {(i i)} {=} \frac {\gamma_ {a}}{2} + \frac {1}{4} \left(\gamma_ {a} - \frac {\gamma_ {a}}{1 6}\right) - 3 c \sqrt {\frac {\eta k \log (1 / \eta \delta)}{N}} K _ {a} ^ {2}
$$
$$
\stackrel {(i i i)} {\geq} \frac {\gamma_ {a}}{4}, \tag {D.33}
$$
where  $(i)$  holds due to (D.8);  $(ii)$  holds since  $\gamma_{a} = \frac{16(\pi - 1)}{k(\pi + k - 1)}$  and  $(iii)$  follows from the condition (D.6). Under the event  $\mathcal{E}_{\mathrm{good}}$ , we know for any  $s \leq \min \{\tau_{+}(\tilde{\phi}^{f}), \widetilde{O}(\eta^{-2})\}$
$$
\begin{array}{l} \left(\boldsymbol {a} _ {s} ^ {i}\right) ^ {\top} \boldsymbol {a} ^ {*} \geq \boldsymbol {a} _ {s} ^ {\top} \boldsymbol {a} ^ {*} - \left| \boldsymbol {a} _ {s} ^ {\top} \boldsymbol {a} ^ {*} - \left(\boldsymbol {a} _ {s} ^ {i}\right) ^ {\top} \boldsymbol {a} ^ {*} \right| \\ \geq \frac {\gamma_ {a}}{4} - c K _ {a} \eta (\sqrt {I k} + I) \sqrt {\log (k / \delta)} \\ = \frac {4 (\pi - 1)}{k (\pi + k - 1)} - c K _ {a} \eta (\sqrt {I k} + I) \sqrt {\log (k / \delta)} \\ \geq \frac {\gamma_ {a}}{8}, \tag {D.34} \\ \end{array}
$$
holds with probability at least  $1 - \delta$ . Recall that  $\tilde{\pmb{v}}_t - \pmb{v}_t = \eta \pmb{h}_{t-1}$ , then it follows that
$$
\begin{array}{l} \left| \left\| \tilde {\boldsymbol {v}} _ {t} \right\| ^ {2} \sin^ {2} \check {\phi} _ {t} - \left\| \boldsymbol {v} _ {t} \right\| ^ {2} \sin^ {2} \phi_ {t} \right| = \left| \left\| \tilde {\boldsymbol {v}} _ {t} \right\| ^ {2} - \left\| \boldsymbol {v} _ {t} \right\| ^ {2} - \left(\left(\tilde {\boldsymbol {v}} _ {t} ^ {\top} \boldsymbol {w} ^ {*}\right) ^ {2} - \left(\boldsymbol {v} _ {t} ^ {\top} \boldsymbol {w} ^ {*}\right) ^ {2}\right) \right| \\ = \left| 2 \eta \boldsymbol {v} _ {t} ^ {\top} \boldsymbol {h} _ {t - 1} + \eta^ {2} \| \boldsymbol {h} _ {t - 1} \| ^ {2} - 2 \eta (\boldsymbol {v} _ {t} ^ {\top} \boldsymbol {w} ^ {*}) (\boldsymbol {h} _ {t - 1} ^ {\top} \boldsymbol {w} ^ {*}) - \eta^ {2} (\boldsymbol {h} _ {t - 1} ^ {\top} \boldsymbol {w} ^ {*}) ^ {2} \right| \\ \leq 2 \eta \left| \mathbf {v} _ {t} ^ {\top} \left(\mathbf {I} - \mathbf {w} ^ {*} (\mathbf {w} ^ {*}) ^ {\top}\right) \mathbf {h} _ {t - 1} \right| + \eta^ {2} \mathbf {h} _ {t - 1} ^ {\top} \left(\mathbf {I} - \mathbf {w} ^ {*} (\mathbf {w} ^ {*}) ^ {\top}\right) \mathbf {h} _ {t - 1} \\ \leq 2 \eta \left\| \left(\mathbf {I} - \boldsymbol {w} ^ {*} (\boldsymbol {w} ^ {*}) ^ {\top}\right) \boldsymbol {v} _ {t} \right\| \| \boldsymbol {h} _ {t - 1} \| + \eta^ {2} \boldsymbol {h} _ {t - 1} ^ {\top} \left(\mathbf {I} - \boldsymbol {w} ^ {*} (\boldsymbol {w} ^ {*}) ^ {\top}\right) \boldsymbol {h} _ {t - 1} \\ \stackrel {(i)} {=} 2 \eta \left(\boldsymbol {v} _ {t} ^ {\top} \left(\mathbf {I} - \boldsymbol {w} ^ {*} (\boldsymbol {w} ^ {*}) ^ {\top}\right) \boldsymbol {v} _ {t}\right) ^ {1 / 2} \| \boldsymbol {h} _ {t - 1} \| + \eta^ {2} \boldsymbol {h} _ {t - 1} ^ {\top} \left(\mathbf {I} - \boldsymbol {w} ^ {*} (\boldsymbol {w} ^ {*}) ^ {\top}\right) \boldsymbol {h} _ {t - 1} \\ = 2 \eta \left(\left\| \boldsymbol {v} _ {t} \right\| ^ {2} - \left(\boldsymbol {v} _ {t} ^ {\top} \boldsymbol {w} ^ {*}\right) ^ {2}\right) ^ {1 / 2} \left\| \boldsymbol {h} _ {t - 1} \right\| + \eta^ {2} \boldsymbol {h} _ {t - 1} ^ {\top} \left(\mathbf {I} - \boldsymbol {w} ^ {*} (\boldsymbol {w} ^ {*}) ^ {\top}\right) \boldsymbol {h} _ {t - 1} \\ \stackrel {(i i)} {=} 2 \eta \sin \phi_ {t} \| \boldsymbol {v} _ {t} \| \| \boldsymbol {h} _ {t - 1} \| + \eta^ {2} \| \boldsymbol {h} _ {t - 1} \| ^ {2}. \tag {D.35} \\ \end{array}
$$
where  $(i)$  holds due to  $\mathbf{I} - \pmb{w}^{*}(\pmb{w}^{*})^{\top}$  is idempotent; and  $(ii)$  holds due to  $\| \pmb{w}^{*}\| = 1$ . Invoking Lemma 1 and (D.26), for any  $s\leq t - 1\leq \min \{\tau_{+}(\tilde{\phi}^{f}),\widetilde{O} (\eta^{-2})\}$ , under the event  $\mathcal{E}_{\mathrm{good}}$ , we can get
$$
\begin{array}{l} \left\| \boldsymbol {v} _ {s} \right\| ^ {2} \sin^ {2} \phi_ {s} = (1 - \eta \lambda_ {s - 1} \cos \phi_ {s - 1}) ^ {2} \left\| \boldsymbol {v} _ {s - 1} \right\| ^ {2} \sin^ {2} \phi_ {s - 1} - 2 \eta M _ {1, s - 1} \\ + \eta^ {2} M _ {2, s - 1} + \| \boldsymbol {v} _ {s} \| ^ {2} \sin^ {2} \phi_ {s} - \| \tilde {\boldsymbol {v}} _ {s} \| ^ {2} \sin^ {2} \check {\phi} _ {s} \\ \stackrel {(i)} {\geq} \| \boldsymbol {v} _ {s - 1} \| ^ {2} \sin^ {2} \phi_ {s - 1} - 2 \eta M _ {1, s - 1} - 6 \eta \| \boldsymbol {h} _ {s - 1} \| - \eta^ {2} \| \boldsymbol {h} _ {s - 1} \| ^ {2} \\ \geq \| \boldsymbol {v} _ {0} \| ^ {2} \sin^ {2} \phi_ {0} - 2 \eta \sum_ {l = 0} ^ {s - 1} M _ {1, l} - \eta \sum_ {l = 0} ^ {s - 1} (6 \| \boldsymbol {h} _ {l} \| + \eta \| \boldsymbol {h} _ {l} \| ^ {2}) \\ \geq \frac {\sin^ {2} \phi_ {0}}{3} - 4 5 0 c \eta (\sqrt {I d} + I) \sqrt {\log (N d / \delta)} K _ {a} ^ {2} - 2 4 c K _ {a} \left(3 + 5 \eta K _ {a} ^ {2}\right) \sqrt {\frac {\eta \log (1 / \delta)}{N}} \\ \geq \frac {\sin^ {2} \phi_ {0}}{6}, \\ \end{array}
$$
where  $(i)$  holds due to (D.34) and  $\cos \phi_s < 0$  for  $s \leq \tau_+(\tilde{\phi}^f)$ ;  $(ii)$  follows from assumption (D.32). It implies that  $\sin^2\phi_s \geq \sin^2\phi_0 / 54$  for any  $s \leq t - 1 \leq \min\{\tau_+(\tilde{\phi}^f), \tilde{O}(\eta^{-2})\}$ . Then we can lower bound  $\sin^2\phi_s^i$  by
$$
\begin{array}{l} \sin^ {2} \phi_ {s} ^ {i} \geq \sin^ {2} \phi_ {s} - \left| \sin^ {2} \phi_ {s} ^ {i} - \sin^ {2} \phi_ {s} \right| \\ \geq \frac {\sin^ {2} \phi_ {0}}{5 4} - 2 \left| \cos \phi_ {s} ^ {i} - \cos \phi_ {s} \right| \\ \geq \frac {\sin^ {2} \phi_ {0}}{5 4} - 4 c \eta K _ {a} ^ {2} (\sqrt {I d} + I) \sqrt {\log (N d / \delta)} \\ \geq \frac {\sin^ {2} \phi_ {0}}{6 4}. \\ \end{array}
$$
Together with the dynamic (D.12) and lower bound (D.34), we can guarantee
$$
\begin{array}{l} \cos \phi_ {t} = \frac {1}{\| \boldsymbol {v} _ {t} \|} \left\{\boldsymbol {v} _ {0} ^ {\top} \boldsymbol {w} ^ {*} + \eta \sum_ {m = t _ {0}} ^ {t - 1} \frac {1}{N} \sum_ {i = 1} ^ {N} \frac {\pi - \phi_ {m} ^ {i}}{2 \pi} \frac {\left(\boldsymbol {a} _ {m} ^ {i}\right) ^ {\top} \boldsymbol {a} ^ {*}}{\| \boldsymbol {v} _ {m} \|} \sin^ {2} \phi_ {m} - \eta \sum_ {m = t _ {0}} ^ {t - 1} \left(\boldsymbol {\xi} _ {m} ^ {\top} \boldsymbol {w} ^ {*} + \boldsymbol {h} _ {m} ^ {\top} \boldsymbol {w} ^ {*}\right) \right\} \\ \geq \frac {1}{\| \boldsymbol {v} _ {t} \|} \left\{- 3 + \eta t \cdot \frac {\gamma_ {a} \sin^ {2} \phi_ {0}}{2 8 8} \cdot \frac {\arcsin (\sin \phi_ {0} / 6)}{2 \pi} \right. \\ \left. - c \eta K _ {a} ^ {2} (\sqrt {I d} + I) \sqrt {\log (N d / \delta)} - c K _ {a} \sqrt {\frac {\eta \log (N d / \eta \delta)}{N}} \right\}. \tag {D.36} \\ \end{array}
$$
Now we take
$$
\tau_ {2, 1} = \frac {1}{\eta} \cdot \frac {5 4 2 9}{\gamma_ {a} \sin^ {2} \phi_ {0}} \frac {1}{\arcsin (\sin \phi_ {0} / 6)} \leq \frac {4 0 0 0 0}{\eta \gamma_ {a}} \frac {1}{\sin^ {3} \phi_ {0}},
$$
where the inequality holds since  $\arcsin (\sin \phi_0 / 6) - \arcsin (0)\geq \sin \phi_0 / 6$  due to  $\arcsin^{\prime}(x) = \frac{1}{\sqrt{1 - x^2}}$ . Plugging it into (D.36), we have
$$
\cos \phi_ {\tau_ {2, 1}} \geq \frac {1}{\| \boldsymbol {v} _ {t} \|} (- 3 + 4 - 1) > \cos \tilde {\phi} ^ {f},
$$
$$
\begin{array}{l} \mathbb {P} \left(\tau_ {2, 1} \leq \tau_ {+} (\tilde {\phi} ^ {f})\right) = \mathbb {P} \left(\left\{\tau_ {2, 1} - 1 \leq \tau_ {+} (\tilde {\phi} ^ {f}) \right\} \cap \left\{\cos \phi_ {\tau_ {2, 1}} \leq \cos \tilde {\phi} ^ {f} \right\}\right) \\ \leq \mathbb {P} \left(\left\{\tau_ {2, 1} - 1 \leq \tau_ {+} (\tilde {\phi} ^ {f}) \right\} \cap \mathcal {E} _ {\text {g o o d}} ^ {c}\right) \leq 5 \delta . \\ \end{array}
$$
Initial angle:  $\phi_0\in (\tilde{\phi}^l,\tilde{\phi}^f ]$  . Now assign  $t_0 = \tau_+$ $(\tilde{\phi}^{f})$  . Denote
$$
\tilde {\mathcal {A}} _ {s} = \left\{\forall t _ {0} \leq m \leq s: \boldsymbol {a} _ {m} ^ {\top} \boldsymbol {a} ^ {*} \geq \frac {\gamma_ {a}}{1 6} \right\}, \quad \text {f o r} s \geq t _ {0}.
$$
Clearly,  $\mathbb{P}(\tilde{\mathcal{A}}_{t_0})\geq 1 - 7\delta$  due to (D.33). Similar to (D.34), we can verify that  $\min_{i\in [N]}(\pmb{a}_m^i)^\top \pmb{a}^*\geq \gamma_a / 32$  for any  $m\leq s$  under  $\tilde{\mathcal{A}}_s\cap \mathcal{E}_{\mathrm{good}}$ . In addition, under the event  $\mathcal{E}_{\mathrm{good}}$ , for any  $\ell \geq 0$  we have
$$
\begin{array}{l} \cos \phi_ {\ell} = \frac {1}{\| \boldsymbol {v} _ {\ell} \|} \left\{\boldsymbol {v} _ {0} ^ {\top} \boldsymbol {w} ^ {*} + \eta \sum_ {m = t _ {0}} ^ {\ell - 1} \frac {1}{N} \sum_ {i = 1} ^ {N} \frac {\pi - \phi_ {m} ^ {i}}{2 \pi} \frac {\left(\boldsymbol {a} _ {m} ^ {i}\right) ^ {\top} \boldsymbol {a} ^ {*}}{\| \boldsymbol {v} _ {m} \|} \sin^ {2} \phi_ {m} - \eta \sum_ {m = t _ {0}} ^ {\ell - 1} \left(\boldsymbol {\xi} _ {m} ^ {\top} \boldsymbol {w} ^ {*} + \boldsymbol {h} _ {m} ^ {\top} \boldsymbol {w} ^ {*}\right) \right\} \\ \geq 6 \cos \tilde {\phi} ^ {f} - 2 c \eta K _ {a} ^ {2} (\sqrt {I d} + I) \sqrt {\log (N d / \delta)} - 2 c K _ {a} \sqrt {\frac {\eta \log (N d / \eta \delta)}{N}} \\ \geq - \frac {6 \pi}{\pi + k - 1} \frac {\left(\mathbf {1} ^ {\top} \mathbf {a} ^ {*}\right) ^ {2}}{k} \log (4 + k ^ {2}) - \frac {2 \pi}{\pi + k - 1} \frac {\left(\mathbf {1} ^ {\top} \mathbf {a} ^ {*}\right) ^ {2}}{k} \log (4 + k ^ {2}) \\ \geq - \frac {1 4 4 \pi}{\pi + k - 1} \frac {\left(\mathbf {1} ^ {\top} \boldsymbol {a} ^ {*}\right) ^ {2}}{k} \log \left(4 + k ^ {2}\right) = \cos \tilde {\phi} ^ {u}. \tag {D.37} \\ \end{array}
$$
It means that  $\tilde{\phi}^l\leq \phi_\ell \leq \tilde{\phi}^f$  for any  $\ell \leq s\leq \min \{\tau_{+}(\tilde{\phi}^{l}),\widetilde{O} (\eta^{-2})\}$  under the event  $\tilde{\mathcal{A}}_s\cap \mathcal{E}_{\mathrm{good}}$ . Hence we know (D.18) holds. Consequently, the lower bound (D.22) also holds for  $\frac{1}{N}\sum_{i = 1}^{N}B_s^i$ . Therefore, similar to (D.33), we can further verify that
$$
\begin{array}{l} \boldsymbol {a} _ {s + 1} ^ {\top} \boldsymbol {a} ^ {*} \geq \frac {\zeta_ {a} \| \boldsymbol {a} ^ {*} \| ^ {2}}{2 (\pi - 1)} \left[ 1 - \left(1 - \frac {\eta (\pi - 1)}{2 \pi}\right) ^ {s} \right] - 3 c \sqrt {\frac {\eta k \log (1 / \eta \delta)}{N}} K _ {a} ^ {2} \\ + \left(\frac {\gamma_ {a}}{4} - \frac {(\pi - 1) \left(\mathbf {1} ^ {\top} \mathbf {a} ^ {*}\right) ^ {2}}{k (\pi + k - 1)}\right) \left(1 - \frac {\eta (\pi - 1)}{2 \pi}\right) ^ {s} \\ \geq \frac {\gamma_ {a}}{1 6}, \\ \end{array}
$$
where we used  $\mathbf{a}_{t_0}^\top \mathbf{a}^* \geq \gamma_a / 4$ . This yields that for any  $t_0 \leq s \leq t - 1$
$$
\tilde {\mathcal {A}} _ {s} \cap \mathcal {E} _ {\text {g o o d}} \cap \left\{t - 1 \leq \tau_ {+} (\tilde {\phi} ^ {l}) \right\} \subseteq \left\{\boldsymbol {a} _ {s + 1} ^ {\top} \boldsymbol {a} ^ {*} \geq \frac {\gamma_ {a}}{1 6} \right\},
$$
$$
\tilde {\mathcal {A}} _ {s} \cap \mathcal {E} _ {\text {g o o d}} \cap \left\{t - 1 \leq \tau_ {+} (\tilde {\phi} ^ {l}) \right\} \subseteq \tilde {\mathcal {A}} _ {s + 1}.
$$
$$
\begin{array}{l} \mathbb {P} \left(\tilde {\mathcal {A}} _ {t} ^ {c} \cap \left\{t - 1 \leq \tau_ {+} (\tilde {\phi} ^ {l}) \right\}\right) = \mathbb {P} \left(\tilde {\mathcal {A}} _ {t - 1} ^ {c} \cap \left\{t - 1 \leq \tau_ {+} (\tilde {\phi} ^ {l}) \right\}\right) \\ + \mathbb {P} \left(\tilde {\mathcal {A}} _ {t - 1} \cap \left\{t - 1 \leq \tau_ {+} \left(\tilde {\phi} ^ {l}\right) \right\} \cap \left\{\boldsymbol {a} _ {s + 1} ^ {\top} \boldsymbol {a} ^ {*} <   \frac {\gamma_ {a}}{1 6} \right\}\right) \\ \leq \mathbb {P} \left(\tilde {\mathcal {A}} _ {t - 1} ^ {c} \cap \left\{t - 1 \leq \tau_ {+} (\tilde {\phi} ^ {l}) \right\}\right) + 5 \delta \\ \leq \mathbb {P} \left(\tilde {\mathcal {A}} _ {t _ {0}} ^ {c} \cap \left\{t - 1 \leq \tau_ {+} (\tilde {\phi} ^ {l}) \right\}\right) + 5 \delta (t - t _ {0}) \\ \leq \mathbb {P} \left(\tilde {\mathcal {A}} _ {t _ {0}} ^ {c}\right) - 5 \delta (t - t _ {0}) \\ \leq 7 \delta + 5 \delta \left(t - t _ {0}\right). \tag {D.38} \\ \end{array}
$$