SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\mathbf {W} _ {j} ^ {(t)} - \mathbf {W} _ {j} ^ {(0)} = \underbrace {\left[ \begin{array}{c c c c} \rho_ {j , 1 , 1} ^ {(t)} \| \mathbf {x} _ {1} \| _ {2} ^ {- 2} & \rho_ {j , 1 , 2} ^ {(t)} \| \mathbf {x} _ {1} \| _ {2} ^ {- 2} & \cdots & \rho_ {j , 1 , n} ^ {(t)} \| \mathbf {x} _ {n} \| _ {2} ^ {- 2} \\ \rho_ {j , 2 , 1} ^ {(t)} \| \mathbf {x} _ {1} \| _ {2} ^ {- 2} & \rho_ {j , 2 , 2} ^ {(t)} \| \mathbf {x} _ {1} \| _ {2} ^ {- 2} & \cdots & \rho_ {j , 2 , n} ^ {(t)} \| \mathbf {x} _ {n} \| _ {2} ^ {- 2} \\ \vdots & \vdots & \ddots & \vdots \\ \rho_ {j , m , 1} ^ {(t)} \| \mathbf {x} _ {1} \| _ {2} ^ {- 2} & \rho_ {j, m, 2} ^ {(t)} \| \mathbf {x} _ {1} \| _ {2} ^ {- 2} & \cdots & \rho_ {j, m, n} ^ {(t)} \| \mathbf {x} _ {n} \| _ {2} ^ {- 2} \end{array} \right]} _ {\mathbf {A} _ {t}} \cdot \underbrace {\left[ \begin{array}{c} \mathbf {x} _ {1} \\ \vdots \\ \mathbf {x} _ {n} \end{array} \right]} _ {: = \mathbf {X}}.
$$
Let  $\mathbf{a}_i(t)^\top$  be the  $i$ -th column of  $\mathbf{A}_t$ . It follows that
$$
\begin{array}{l} \left\| \mathbf {W} _ {j} ^ {(t)} - \mathbf {W} _ {j} ^ {(0)} \right\| _ {F} ^ {2} = \operatorname {T r} \left(\mathbf {A} _ {t} \mathbf {X} \mathbf {X} ^ {\top} \mathbf {A} _ {t} ^ {\top}\right) \\ = \operatorname {T r} \left(\left(\sum_ {i = 1} ^ {n} \mathbf {a} _ {i} (t) ^ {\top} \mathbf {x} _ {i}\right) \left(\sum_ {i = 1} ^ {n} \mathbf {x} _ {i} ^ {\top} \mathbf {a} _ {i} (t)\right)\right) \\ = \operatorname {T r} \left(\sum_ {i = 1} ^ {n} \sum_ {i ^ {\prime} = 1} ^ {n} \mathbf {a} _ {i} (t) ^ {\top} \mathbf {x} _ {i} \mathbf {x} _ {i ^ {\prime}} ^ {\top} \mathbf {a} _ {i ^ {\prime}} (t)\right) \\ = \sum_ {i = 1} ^ {n} \sum_ {i ^ {\prime} = 1} ^ {n} \left\langle \mathbf {x} _ {i}, \mathbf {x} _ {i ^ {\prime}} \right\rangle \cdot \operatorname {T r} \left(\mathbf {a} _ {i} (t) ^ {\top} \mathbf {a} _ {i ^ {\prime}} (t)\right) \\ \end{array}
$$
$$
\begin{array}{l} = \sum_ {i = 1} ^ {n} \| \mathbf {x} _ {i} \| _ {2} ^ {2} \cdot \operatorname {T r} \left(\mathbf {a} _ {i} (t) ^ {\top} \mathbf {a} _ {i} (t)\right) + \sum_ {i \neq i ^ {\prime}} \left\langle \mathbf {x} _ {i}, \mathbf {x} _ {i ^ {\prime}} \right\rangle \cdot \operatorname {T r} \left(\mathbf {a} _ {i} (t) ^ {\top} \mathbf {a} _ {i ^ {\prime}} (t)\right) \\ \leq R _ {\max } ^ {2} \sum_ {i = 1} ^ {n} \operatorname {T r} \left(\mathbf {a} _ {i} (t) ^ {\top} \mathbf {a} _ {i} (t)\right) + p \sum_ {i \neq i ^ {\prime}} | \operatorname {T r} \left(\mathbf {a} _ {i} (t) ^ {\top} \mathbf {a} _ {i ^ {\prime}} (t)\right) |, \\ \end{array}
$$
$$
\operatorname {T r} \left(\mathbf {a} _ {i} (t) ^ {\top} \mathbf {a} _ {i} (t)\right) = \sum_ {r = 1} ^ {m} \left([ \mathbf {a} _ {i} (t) ] _ {r}\right) ^ {2} = \sum_ {r = 1} ^ {m} \left(\rho_ {j, r, i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2}\right) ^ {2} \leq \sum_ {r = 1} ^ {m} \left(\rho_ {j, r, i} ^ {(t)}\right) ^ {2} R _ {\min } ^ {- 4} \leq C m R _ {\min } ^ {- 4} (\log (t)) ^ {2},
$$
$$
\left| \operatorname {T r} \left(\mathbf {a} _ {i} (t) ^ {\top} \mathbf {a} _ {i ^ {\prime}} (t)\right) \right| \leq \sum_ {r = 1} ^ {m} \left| \left[ \mathbf {a} _ {i} (t) \right] _ {r} \left[ \mathbf {a} _ {i ^ {\prime}} (t) \right] _ {r} \right| = \sum_ {r = 1} ^ {m} \left| \rho_ {j, r, i} ^ {(t)} \rho_ {j, r, i ^ {\prime}} ^ {(t)} \right| \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \leq C m R _ {\min } ^ {- 4} (\log (t)) ^ {2}.
$$
Accordingly, we have
$$
\begin{array}{l} \left\| \mathbf {W} _ {j} ^ {(t)} - \mathbf {W} _ {j} ^ {(0)} \right\| _ {F} ^ {2} \leq C m n R _ {\max } ^ {2} R _ {\min } ^ {- 4} (\log (t)) ^ {2} + C m n ^ {2} p R _ {\min } ^ {- 4} (\log (t)) ^ {2} \\ = C m n R _ {\max } ^ {2} R _ {\min } ^ {- 4} \left(1 + R _ {\max } ^ {- 2} n p\right) \left(\log (t)\right) ^ {2} \\ \leq C ^ {\prime} m n R _ {\max } ^ {2} R _ {\min } ^ {- 4} (\log (t)) ^ {2}. \\ \end{array}
$$
On the other hand, we will give an lower bound for  $\| \mathbf{W}_j^{(t)} - \mathbf{W}_j^{(0)}\| _2$
$$
\begin{array}{l} \| \mathbf {W} _ {j} ^ {(t)} - \mathbf {W} _ {j} ^ {(0)} \| _ {2} = \max  _ {\mathbf {y} \in S ^ {d - 1}} \| (\mathbf {W} _ {j} ^ {(t)} - \mathbf {W} _ {j} ^ {(0)}) \mathbf {y} \| _ {2} \\ \geq \frac {\left\| \left(\mathbf {W} _ {j} ^ {(t)} - \mathbf {W} _ {j} ^ {(0)}\right) \mathbf {X} ^ {\top} \left(\mathbf {X X} ^ {\top}\right) ^ {- 1} \mathbf {1} _ {n} \right\| _ {2}}{\left\| \mathbf {X} ^ {\top} \left(\mathbf {X X} ^ {\top}\right) ^ {- 1} \mathbf {1} _ {n} \right\| _ {2}} \\ = \frac {\left\| \mathbf {A} _ {t} \mathbf {1} _ {n} \right\| _ {2}}{\left\| \mathbf {X} ^ {\top} (\mathbf {X X} ^ {\top}) ^ {- 1} \mathbf {1} _ {n} \right\| _ {2}}. \\ \end{array}
$$
We first provide a lower bound for  $\| \mathbf{A}_t\mathbf{1}_n\| _2$  . Note that
$$
\mathbf {A} _ {t} \mathbf {1} _ {n} = \left[ \begin{array}{c} \sum_ {i = 1} ^ {n} \rho_ {j, 1, i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \\ \vdots \\ \sum_ {i = 1} ^ {n} \rho_ {j, m, i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \end{array} \right],
$$
we need to bound  $\sum_{i=1}^{n} \rho_{j,r,i}^{(t)}\|\mathbf{x}_i\|_2^{-2}, r \in [m]$ . By Corollary E.3, there exists time  $T$  such that for any  $t \geq T$ ,  $|\underline{\rho}_{y_i,r,i'}^{(t)}| \leq cR_{\min}^{-2}pn\overline{\rho}_{y_i,r,i}'$ ,  $\forall i \in S_r^{(0)}$  and  $\forall i' \in [n]$  with  $y_{i'} = -y_i$ . Therefore, we have for  $t \geq T$  that
$$
\begin{array}{l} \sum_ {r = 1} ^ {m} \sum_ {i = 1} ^ {n} \rho_ {j, r, i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} = \sum_ {r = 1} ^ {m} \left(\sum_ {i \in S _ {j}} \overline {{\rho}} _ {j, r, i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} + \sum_ {i \in S _ {- j}} \underline {{\rho}} _ {j, r, i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2}\right) \\ = \sum_ {i \in S _ {j}} \sum_ {r = 1} ^ {m} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} + \sum_ {i \in S _ {- j}} \sum_ {r = 1} ^ {m} \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \\ \geq \sum_ {i \in S _ {j}} \sum_ {r = 1} ^ {m} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} R _ {\max } ^ {- 2} + \sum_ {i \in S _ {- j}} \sum_ {r = 1} ^ {m} \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} R _ {\min } ^ {- 2} \\ \geq \sum_ {i \in S _ {j}} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} R _ {\max } ^ {- 2} - \frac {m | S _ {- j} |}{| S _ {j} |} \sum_ {i \in S _ {j}} \frac {c R _ {\min } ^ {- 2} p n}{| S _ {i} ^ {(0)} |} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} R _ {\min } ^ {- 2} \\ \geq \sum_ {i \in S _ {j}} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} R _ {\max } ^ {- 2} - \frac {m | S _ {- j} | \cdot c R _ {\min } ^ {- 4} R _ {\max } ^ {2} p n}{| S _ {j} | \cdot \min  _ {i \in S _ {j}} | S _ {i} ^ {(0)} |} \sum_ {i \in S _ {j}} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} R _ {\max } ^ {- 2} \\ \end{array}
$$
$$
\geq \frac {R _ {\max } ^ {- 2}}{2} \sum_ {i \in S _ {j}} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {j, r, i} ^ {(t)}, \tag {F.2}
$$
where the second inequality is by Corollary E.3 and hence
$$
| \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} | \leq c R _ {\min } ^ {- 2} p n \overline {{\rho}} _ {y _ {i ^ {\prime}}, r ^ {\prime}, i ^ {\prime}} ^ {(t)}, \forall r, r ^ {\prime} \in [ m ], \forall i, i ^ {\prime} \in [ n ],
$$
$$
| \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} | \leq \frac {c R _ {\mathrm {m i n}} ^ {- 2} p n}{| S _ {i ^ {\prime}} ^ {(0)} |} \sum_ {r \in S _ {i ^ {\prime}} ^ {(0)}} \overline {{\rho}} _ {y _ {i ^ {\prime}}, r, i ^ {\prime}} ^ {(t)}, \forall r \in [ m ], \forall i, i ^ {\prime} \in [ n ],
$$
$$
| \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} | \leq \frac {1}{| S _ {j} |} \sum_ {i \in S _ {j}} \frac {c R _ {\operatorname* {m i n}} ^ {- 2} p n}{| S _ {i} ^ {(0)} |} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)}, \forall r \in [ m ], \forall i \in [ n ],
$$
the last inequality is by
$$
\frac {m | S _ {- j} | \cdot c R _ {\mathrm {m i n}} ^ {- 4} R _ {\mathrm {m a x}} ^ {2} p n}{| S _ {j} | \cdot \mathrm {m i n} _ {i \in S _ {j}} | S _ {i} ^ {(0)} |} \leq c ^ {\prime} c R _ {\mathrm {m i n}} ^ {- 4} R _ {\mathrm {m a x}} ^ {2} p n \cdot \frac {| S _ {- j} |}{| S _ {j} |} \leq \frac {1}{2}.
$$
Then, we have for  $t \geq T$  that
$$
\begin{array}{l} \| \mathbf {A} _ {t} \mathbf {1} _ {n} \| _ {2} = \sqrt {\sum_ {r = 1} ^ {m} \left(\sum_ {i = 1} ^ {n} \rho_ {j , r , i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2}\right) ^ {2}} \\ \geq \left| \sum_ {r = 1} ^ {m} \sum_ {i = 1} ^ {n} \rho_ {j, r, i} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} / \sqrt {m} \right| \\ \geq \frac{R_{\max}^{-2}}{2\sqrt{m}}\sum_{i\in S_{j}}\sum_{r\in S_{i}^{(0)}}\overline{\rho}_{j,r,i}^{(t)} \\ \geq \frac {R _ {\max } ^ {- 2} | S _ {j} | | S _ {i} ^ {(0)} |}{2 \sqrt {m}} \cdot \log \left(1 + \frac {\eta R _ {\min } ^ {2} e ^ {- \beta}}{2 c ^ {\prime} n m} \cdot t\right) \\ \geq C R _ {\max} ^ {- 2} \sqrt {m} n \log (t) \\ \end{array}
$$
where the second inequality is by (F.2); the third inequality is by the second bullet of Lemma E.1. For  $\| \mathbf{X}^{\top}(\mathbf{X}\mathbf{X}^{\top})^{-1}\mathbf{1}_n\| _2$ , we have
$$
\begin{array}{l} \| \mathbf {X} ^ {\top} (\mathbf {X X} ^ {\top}) ^ {- 1} \mathbf {1} _ {n} \| _ {2} = \sqrt {\mathbf {1} _ {n} ^ {\top} (\mathbf {X X} ^ {\top}) ^ {- 1} \mathbf {X X} ^ {\top} (\mathbf {X X} ^ {\top}) ^ {- 1} \mathbf {1} _ {n}} \\ = \sqrt {\mathbf {1} _ {n} ^ {\top} (\mathbf {X X} ^ {\top}) ^ {- 1} \mathbf {1} _ {n}} \\ \leq \frac {\| \mathbf {1} _ {n} \| _ {2}}{\sqrt {\lambda_ {\operatorname* {m i n}} (\mathbf {X X} ^ {\top})}} \\ \end{array}
$$
By the Gershgorin circle theorem, we know that  $\lambda_{\mathrm{min}}(\mathbf{X}\mathbf{X}^{\top})$  lies within at least one of the Gershgorin discs  $D((\mathbf{X}\mathbf{X}^{\top})_{ii},R_i),i\in [n]$  where  $D((\mathbf{X}\mathbf{X}^{\top})_{ii},R_i)$  is a closed disc centered at  $(\mathbf{X}\mathbf{X}^{\top})_{ii} = \|\mathbf{x}_i\|_2^2$  with radius  $R_i = \sum_{i'\neq i}|(\mathbf{X}\mathbf{X}^{\top})_{ii'}| = \sum_{i'\neq i}|\langle \mathbf{x}_i,\mathbf{x}_{i'}\rangle |.$  Assume  $\lambda_{\mathrm{min}}(\mathbf{X}\mathbf{X}^{\top})$  lies within  $D((\mathbf{X}\mathbf{X}^{\top})_{ii},R_i)$ , then we can get following lower bound for  $\lambda_{\mathrm{min}}(\mathbf{X}\mathbf{X}^{\top})$ :
$$
\lambda_ {\mathrm {m i n}} (\mathbf {X X} ^ {\top}) \geq \| \mathbf {x} _ {i} \| _ {2} ^ {2} - \sum_ {i ^ {\prime} \neq i} | \langle \mathbf {x} _ {i}, \mathbf {x} _ {i ^ {\prime}} \rangle | \geq R _ {\mathrm {m i n}} ^ {2} - n p = (1 - R _ {\mathrm {m i n}} ^ {- 2} n p) R _ {\mathrm {m i n}} ^ {2} \geq R _ {\mathrm {m i n}} ^ {2} / 2.
$$
$$
\| \mathbf {X} ^ {\top} (\mathbf {X X} ^ {\top}) ^ {- 1} \mathbf {1} _ {n} \| _ {2} \leq \frac {\| \mathbf {1} _ {n} \| _ {2}}{\sqrt {\lambda_ {\operatorname* {m i n}} (\mathbf {X X} ^ {\top})}} \leq \frac {\sqrt {2 n}}{R _ {\operatorname* {m i n}}}.
$$
Accordingly,
$$
\| \mathbf {W} _ {j} ^ {(t)} - \mathbf {W} _ {j} ^ {(0)} \| _ {2} \geq \frac {\| \mathbf {A} _ {t} \mathbf {1} _ {n} \| _ {2}}{\| \mathbf {X} ^ {\top} (\mathbf {X X} ^ {\top}) ^ {- 1} \mathbf {1} _ {n} \| _ {2}} \geq \frac {C R _ {\max} ^ {- 2} \sqrt {m n} \log (t)}{\sqrt {2 n} / R _ {\min}} = C ^ {\prime \prime} R _ {\max} ^ {- 2} R _ {\min} \sqrt {m n} \log (t).
$$
Therefore, we have for  $t \geq T$  that
$$
\frac {\| \mathbf {W} _ {j} ^ {(t)} - \mathbf {W} _ {j} ^ {(0)} \| _ {F} ^ {2}}{\| \mathbf {W} _ {j} ^ {(t)} - \mathbf {W} _ {j} ^ {(0)} \| _ {2} ^ {2}} \leq \frac {C ^ {\prime} m n R _ {\max} ^ {2} R _ {\min} ^ {- 4} (\log (t)) ^ {2}}{C ^ {\prime \prime 2} m n R _ {\max} ^ {- 4} R _ {\min} ^ {2} (\log (t)) ^ {2}} \leq \frac {C ^ {\prime} R _ {\max} ^ {6}}{C ^ {\prime \prime 2} R _ {\min} ^ {6}},
$$
Next, we will provide an example of training data satisfying the condition in Theorem 4.3 and prove that the stable rank of  $\mathbf{W}_j^{(t)}$  trained by gradient descent using such data will converge to  $2\pm o(1)$ . We first provide the following lemma about the increasing rate of coefficients  $\rho_{j,r,i}^{(t)}$ .
Lemma F.2. If training data  $\mathbf{x}_1,\dots ,\mathbf{x}_n$  are mutually orthogonal, the activation pattern after time  $T = C\eta^{-1}R_{\mathrm{min}}^{-2} / nm$  is determined by the activation pattern at initialization, i.e.,
$$
\langle \mathbf {w} _ {y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle \geq 0, \quad \text {i f} \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle \geq 0,
$$
$$
\langle \mathbf {w} _ {y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle <   0, \qquad \qquad \text {i f} \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle <   0,
$$
$$
\langle \mathbf {w} _ {- y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle <   0, \qquad \qquad \text {i f} \langle \mathbf {w} _ {- y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle \geq 0,
$$
$$
\langle \mathbf {w} _ {- y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle <   0, \qquad \qquad \text {i f} \langle \mathbf {w} _ {- y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle <   0,
$$
for any time  $t \geq T$ . Besides,  $\rho_{j,r,i}^{(t)}$  satisfy the following properties:
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} = 0, \forall t \geq 0, \quad \text {i f} \langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \rangle <   0,
$$
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} = \overline {{\rho}} _ {y _ {i}, r, i} ^ {(T)}, \forall t \geq T, \quad \text {i f} \langle \mathbf {w} _ {- y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle \geq 0,
$$
$$
\lim  _ {t \to \infty} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} / \log t = m / | S _ {i} ^ {(0)} |, \qquad \qquad \text {i f} \langle \mathbf {w} _ {y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle \geq 0.
$$
Proof of Lemma F.2. Part 1. For if  $\langle \mathbf{w}_{j,r}^{(0)},\mathbf{x}_i\rangle < 0$  we first prove by induction that
$$
\rho_ {j, r, i} ^ {(t)} = 0, \left\langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \right\rangle = \left\langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \right\rangle <   0, \forall t \geq 0. \tag {F.3}
$$
The result is obvious at  $t = 0$  as all the coefficients are zero. Suppose that there exists  $\widetilde{t}$  such that (F.3) holds for all time  $0 \leq t \leq \widetilde{t} - 1$ . We aim to prove that (F.3) also holds for  $t = \widetilde{t}$ . Recall that by (5.4), (5.5) and with (F.3) at time  $\widetilde{t} - 1$ , we have
$$
\overline {{\rho}} _ {j, r, i} ^ {(\widetilde {t})} = \overline {{\rho}} _ {j, r, i} ^ {(\widetilde {t} - 1)} - \frac {\eta}{n m} \cdot \ell_ {i} ^ {(\widetilde {t} - 1)} \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {j, r} ^ {(\widetilde {t} - 1)}, \mathbf {x} _ {i} \rangle) \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \cdot \mathbb {1} (y _ {i} = j) = \overline {{\rho}} _ {j, r, i} ^ {(\widetilde {t} - 1)} = 0,
$$
$$
\underline {{\rho}} _ {j, r, i} ^ {(\widetilde {t})} = \underline {{\rho}} _ {j, r, i} ^ {(\widetilde {t} - 1)} + \frac {\eta}{n m} \cdot \ell_ {i} ^ {(\widetilde {t} - 1)} \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {j, r} ^ {(\widetilde {t} - 1)}, \mathbf {x} _ {i} \rangle) \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \cdot \mathbb {1} (y _ {i} = - j) = \underline {{\rho}} _ {j, r, i} ^ {(\widetilde {t} - 1)} = 0.
$$
By (5.1) and the orthogonality of training data, we can get
$$
\langle \mathbf {w} _ {j, r} ^ {(\widetilde {t})}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \sum_ {i ^ {\prime} = 1} ^ {n} \rho_ {j, r, i ^ {\prime}} ^ {(\widetilde {t})} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \rho_ {j, r, i} ^ {(\widetilde {t})} = \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle <   0.
$$
Therefore, (F.3) holds at time  $\widetilde{t}$ , which completes the induction.
For if  $\langle \mathbf{w}_{j,r}^{(0)},\mathbf{x}_i\rangle \geq 0$  and  $j = y_{i}$  , we will next prove by induction that
$$
\bar {\rho} _ {j, r, i} ^ {(t)} \geq 0, \langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \rangle \geq \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle \geq 0, \forall t \geq 0. \tag {F.4}
$$
The result is natural at  $t = 0$ . Suppose that there exists  $\widetilde{t}$  such that (F.4) hold for all time  $0 \leq t \leq \widetilde{t} - 1$ . By (5.4) and (F.4) at time  $\widetilde{t} - 1$ , we have
$$
\overline {{\rho}} _ {j, r, i} ^ {(\widetilde {t})} = \overline {{\rho}} _ {j, r, i} ^ {(\widetilde {t} - 1)} + \frac {\eta}{n m} \cdot | \ell_ {i} ^ {\prime (\widetilde {t} - 1)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \geq \overline {{\rho}} _ {j, r, i} ^ {(\widetilde {t} - 1)} \geq 0
$$
and hence the orthogonality of training data, we can get
$$
\langle \mathbf {w} _ {j, r} ^ {(\widetilde {t})}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \rho_ {j, r, i} ^ {(\widetilde {t})} = \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle \geq 0.
$$
Therefore, (F.4) hold at time  $\widetilde{t}$ , which completes the induction.
For if  $\langle \mathbf{w}_{j,r}^{(0)},\mathbf{x}_i\rangle \geq 0$  and  $j = -y_{i}$ , we first show that under the same condition as Theorem 4.3 it holds that
$$
\underline {{\rho}} _ {j, r, i} ^ {(T)} <   - \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle .
$$
Since  $T = C\eta^{-1}R_{\min}^{-2} / nm$ , we have  $\max_{j,r,i}\{|\rho_{j,r,i}^{(t)}| = O(1)$  for  $t \leq T$ . Therefore, we know that  $F_{+1}(\mathbf{W}_{+1}^{(t)},\mathbf{x}_i),F_{-1}(\mathbf{W}_{-1}^{(t)},\mathbf{x}_i) = O(1)$ . Thus there exists a positive constant  $c$  such that  $-\ell_i^{\prime (t)} \geq c$  for all  $i \in [n]$ . Here we use the method of proof by contradiction. Assume  $\underline{\rho}_{j,r,i}^{(T)} \geq -\langle \mathbf{w}_{j,r}^{(0)},\mathbf{x}_i\rangle$ . Since  $\underline{\rho}_{j,r,i}^{(T)} \leq \underline{\rho}_{j,r,i}^{(t)}$  for  $0 \leq t \leq T$  which can be seen from (5.5), we have  $\underline{\rho}_{j,r,i}^{(t)} \geq -\langle \mathbf{w}_{j,r}^{(0)},\mathbf{x}_i\rangle$  for all  $t \leq T$ . Then, we can get
$$
\langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \rho_ {j, r, i} ^ {(t)} \geq 0, \forall t \leq T.
$$
Therefore, by the non-negativeness of  $\langle \mathbf{w}_{j,r}^{(t)},\mathbf{x}_i\rangle$  and (5.5), we can get
$$
| \underline {{\rho}} _ {j, r, i} ^ {(t + 1)} | = | \underline {{\rho}} _ {j, r, i} ^ {(t)} | + \frac {\eta}{n m} \cdot | \ell_ {i} ^ {\prime (t)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \geq | \underline {{\rho}} _ {j, r, i} ^ {(t)} | + \frac {c \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2}}{n m}
$$
$$
| \underline {{\rho}} _ {j, r, i} ^ {(T)} | \geq \frac {c \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} T}{n m} = c C \| \mathbf {x} _ {i} \| _ {2} ^ {2} R _ {\mathrm {m i n}} ^ {- 2} \geq c C \geq \beta ,
$$
which is a contradiction. Therefore,  $\underline{\rho}_{j,r,i}^{(T)} < -\langle \mathbf{w}_{j,r}^{(0)}, \mathbf{x}_i \rangle$ . By (5.5), we have  $\underline{\rho}_{j,r,i}^{(t)} \leq \underline{\rho}_{j,r,i}^{(T)} < -\langle \mathbf{w}_{j,r}^{(0)}, \mathbf{x}_i \rangle$  for  $t \geq T$ . Therefore,
$$
\langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \rho_ {j, r, i} ^ {(t)} <   0, \forall t \geq T.
$$
Plugging this into (5.5) gives us
$$
\underline {{\rho}} _ {j, r, i} ^ {(t + 1)} = \underline {{\rho}} _ {j, r, i} ^ {(t)} + \frac {\eta}{n m} \cdot \ell_ {i} ^ {(t)} \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \rangle) \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} = \underline {{\rho}} _ {j, r, i} ^ {(t)}, \forall t \geq T.
$$
This completes the proof of the first half of the lemma about the activation pattern as well as the first two properties of  $\rho_{j,r,i}^{(t)}$ .
Part 2. Now we will show that
$$
\lim  _ {t \rightarrow \infty} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} / \log t = m / | S _ {i} ^ {(0)} |, \tag {F.5}
$$
if  $\langle \mathbf{w}_{y_i,r}^{(t)},\mathbf{x}_i\rangle \geq 0$  . By the activation pattern, we can get
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t + 1)} = \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} + \frac {\eta}{n m} \cdot | \ell_ {i} ^ {\prime (t)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \forall t \geq 0, \mathrm {f o r} \langle \mathbf {w} _ {y _ {i}, i} ^ {(t)}, \mathbf {x} _ {i} \rangle \geq 0,
$$
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(t)} = 0, \quad \forall t \geq 0, \quad \text {f o r} \langle \mathbf {w} _ {y _ {i}, i} ^ {(t)}, \mathbf {x} _ {i} \rangle <   0, \tag {F.6}
$$
$$
\underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} = \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(T)}, \quad \forall t \geq 0, \quad \text {f o r} \langle \mathbf {w} _ {- y _ {i}, i} ^ {(t)}, \mathbf {x} _ {i} \rangle \geq 0,
$$
$$
\underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} = 0, \quad \forall t \geq 0, \quad \text {f o r} \langle \mathbf {w} _ {- y _ {i}, i} ^ {(t)}, \mathbf {x} _ {i} \rangle <   0.
$$
Given this activation pattern, we can get for  $t \geq 0$  that
$$
\begin{array}{l} F _ {y _ {i}} (\mathbf {W} _ {y _ {i}} ^ {(t)}, \mathbf {x} _ {i}) - F _ {- y _ {i}} (\mathbf {W} _ {- y _ {i}} ^ {(t)}, \mathbf {x} _ {i}) = \frac {1}{m} \sum_ {r = 1} ^ {m} \sigma (\langle \mathbf {w} _ {y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle) - \frac {1}{m} \sum_ {r = 1} ^ {m} \sigma (\langle \mathbf {w} _ {- y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle) \\ \leq \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \langle \mathbf {w} _ {y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle \\ = \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} [ \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} ] \\ \leq \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} + \beta , \\ \end{array}
$$
$$
\begin{array}{l} F _ {y _ {i}} \left(\mathbf {W} _ {y _ {i}} ^ {(t)}, \mathbf {x} _ {i}\right) - F _ {- y _ {i}} \left(\mathbf {W} _ {- y _ {i}} ^ {(t)}, \mathbf {x} _ {i}\right) = \frac {1}{m} \sum_ {r = 1} ^ {m} \sigma \left(\left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \right\rangle\right) - \frac {1}{m} \sum_ {r = 1} ^ {m} \sigma \left(\left\langle \mathbf {w} _ {- y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \right\rangle\right) \\ \geq \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \right\rangle - \beta \\ = \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} [ \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} ] - \beta \\ \geq \frac{1}{m}\sum_{r\in S_{i}^{(0)}}\overline{\rho}_{y_{i},r,i}^{(t)} - 2\beta . \\ \end{array}
$$
Therefore, we can get following upper and lower bounds for  $|\ell_i^{\prime (t)}|:$
$$
| \ell_ {i} ^ {\prime (t)} | \leq \exp \left(- \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} + 2 \beta\right), \forall t \geq 0,
$$
$$
| \ell_ {i} ^ {\prime (t)} | \geq \frac {1}{2} \exp \left(- \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \beta\right), \forall t \geq 0.
$$
And it follows that
$$
\frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t + 1)} \leq \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {2 \beta}}{n m} \cdot \exp \left(- \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)}\right), \forall t \geq 0,
$$
$$
\frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t + 1)} \geq \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {- \beta}}{2 n m} \cdot \exp \left(- \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)}\right), \forall t \geq 0.
$$
By leveraging Lemma H.1 as well as Lemma H.2 and taking
$$
x _ {t} = \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)},
$$