SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\mathbf {w} _ {j, r} ^ {(t)} = \mathbf {w} _ {j, r} ^ {(0)} + \underbrace {\sum_ {i = 1} ^ {n} \rho_ {j , r , i} ^ {(t)} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \cdot \mathbf {x} _ {i}} _ {: := \mathbf {v} _ {j, r} ^ {(t)}}.
$$
We first show that  $\| \mathbf{v}_{j,r}^{(t)}\| _2 = \Theta (\log t)$
$$
\begin{array}{l} \| \mathbf {v} _ {j, r} ^ {(t)} \| _ {2} ^ {2} = \left(\sum_ {i = 1} ^ {n} \rho_ {j, r, i} ^ {(t)} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \cdot \mathbf {x} _ {i}\right) ^ {2} \\ = \sum_ {i = 1} ^ {n} (\rho_ {j, r, i} ^ {(t)}) ^ {2} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} + \sum_ {i \neq i ^ {\prime}} \rho_ {j, r, i} ^ {(t)} \rho_ {j, r, i ^ {\prime}} ^ {(t)} \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \langle \mathbf {x} _ {i}, \mathbf {x} _ {i ^ {\prime}} \rangle \\ \geq \sum_ {i = 1} ^ {n} (\rho_ {j, r, i} ^ {(t)}) ^ {2} \cdot R _ {\max} ^ {- 2} - \sum_ {i \neq i ^ {\prime}} | \rho_ {j, r, i} ^ {(t)} | | \rho_ {j, r, i ^ {\prime}} ^ {(t)} | \cdot R _ {\min} ^ {- 4} p \\ \geq R _ {\max } ^ {- 2} \left(1 - R ^ {2} R _ {\min } ^ {- 2} c _ {1} ^ {- 1} \gamma^ {- 4} n p\right) \sum_ {i = 1} ^ {n} \left(\rho_ {j, r, i} ^ {(t)}\right) ^ {2} \\ = \Theta (n R _ {\mathrm {m a x}} ^ {- 2} \log^ {2} (t)), \\ \end{array}
$$
where the second last inequality is by triangle inequality; the last inequality is by  $|\rho_{j,r,i}^{(t)}| \leq \gamma^{-4}c_1^{-1}|\rho_{j,r,i'}^{(t)}|$  from Lemma C.1 and Lemma C.4.
By the definition of  $\mathbf{v}_{j,r}^{(t)}$ , we have
$$
\begin{array}{l} \| \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, r ^ {\prime}} ^ {(t)} \| _ {2} ^ {2} = \left\| \sum_ {i = 1} ^ {n} \rho_ {j, r, i} ^ {(t)} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \cdot \mathbf {x} _ {i} - \sum_ {i = 1} ^ {n} \rho_ {j, r ^ {\prime}, i} ^ {(t)} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \cdot \mathbf {x} _ {i} \right\| _ {2} ^ {2} \\ = \left\| \sum_ {i = 1} ^ {n} (\rho_ {j, r, i} ^ {(t)} - \rho_ {j, r ^ {\prime}, i} ^ {(t)}) \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \cdot \mathbf {x} _ {i} \right\| _ {2} ^ {2} \\ = \sum_ {i = 1} ^ {n} (\rho_ {j, r, i} ^ {(t)} - \rho_ {j, r ^ {\prime}, i} ^ {(t)}) ^ {2} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} + \sum_ {i \neq i ^ {\prime}} (\rho_ {j, r, i} ^ {(t)} - \rho_ {j, r ^ {\prime}, i} ^ {(t)}) (\rho_ {j, r, i ^ {\prime}} ^ {(t)} - \rho_ {j, r ^ {\prime}, i ^ {\prime}} ^ {(t)}) \frac {\langle \mathbf {x} _ {i} , \mathbf {x} _ {i ^ {\prime}} \rangle}{\| \mathbf {x} _ {i} \| _ {2} ^ {2} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {2}} \\ \leq (C ^ {\prime}) ^ {2} n R _ {\mathrm {m i n}} ^ {- 2} + (C ^ {\prime}) ^ {2} n ^ {2} R _ {\mathrm {m i n}} ^ {- 4} p \\ \leq 2 (C ^ {\prime}) ^ {2} n R _ {\mathrm {m i n}} ^ {- 2}, \\ \end{array}
$$
where the first inequality is by Lemma D.1 and Lemma D.2.
Now, we are ready to estimate the stable rank of  $\mathbf{W}^{(t)}$ . On the one hand, for  $\| \mathbf{W}_j^{(t)}\| _F^2$ , we have
$$
\begin{array}{l} \| \mathbf {W} _ {j} ^ {(t)} \| _ {F} ^ {2} = \sum_ {r} \| \mathbf {w} _ {j, r} ^ {(t)} \| _ {2} ^ {2} \\ = \sum_ {r} \| \mathbf {w} _ {j, r} ^ {(0)} + \mathbf {v} _ {j, r} ^ {(t)} \| _ {2} ^ {2} \\ = \sum_ {r} \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} ^ {2} + \| \mathbf {v} _ {j, r} ^ {(t)} \| _ {2} ^ {2} + 2 \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {v} _ {j, r} ^ {(t)} \rangle \\ \leq \sum_ {r} \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} ^ {2} + \left(\| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} + \| \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2}\right) ^ {2} + 2 \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} \left(\| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} + \| \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2}\right) \\ = m \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} ^ {2} + 2 \left(\sum_ {r} \| \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} + \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2}\right) \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} \\ + \left(\sum_ {r} \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} ^ {2} + \| \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} ^ {2} + 2 \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} \| \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2}\right) \\ \leq m \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} ^ {2} + m C _ {1} \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} + m C _ {2}. \\ \end{array}
$$
where the first inequality is by triangle inequality and Cauchy inequality; the last inequality is by Lemma D.1, Lemma D.2 and taking
$$
C _ {1} = 3 \left(\sigma_ {0} \sqrt {d} + C ^ {\prime} \sqrt {n} R _ {\min } ^ {- 1}\right),
$$
$$
C _ {2} = 2 (\sigma_ {0} \sqrt {d} + C ^ {\prime} \sqrt {n} R _ {\mathrm {m i n}} ^ {- 1}) ^ {2}.
$$
On the other hand, for  $\| \mathbf{W}_j^{(t)}\| _2^2$  , we have
$$
\begin{array}{l} \| \mathbf {W} _ {j} ^ {(t)} \| _ {2} ^ {2} = \max  _ {\mathbf {x} \in S ^ {d - 1}} \| \mathbf {W} _ {j} ^ {(0)} \mathbf {x} + \mathbf {V} _ {j} ^ {(t)} \mathbf {x} \| _ {2} ^ {2} \\ = \max  _ {\mathbf {x} \in S ^ {d - 1}} \| \mathbf {W} _ {j} ^ {(0)} \mathbf {x} \| _ {2} ^ {2} + \| \mathbf {V} _ {j} ^ {(t)} \mathbf {x} \| _ {2} ^ {2} + 2 \langle \mathbf {W} _ {j} ^ {(0)} \mathbf {x}, \mathbf {V} _ {j} ^ {(t)} \mathbf {x} \rangle \\ = \max  _ {\mathbf {x} \in S ^ {d - 1}} \sum_ {r} \left\langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} \right\rangle^ {2} + \sum_ {r} \left\langle \mathbf {v} _ {j, r} ^ {(t)}, \mathbf {x} \right\rangle^ {2} + \sum_ {r} \left\langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} \right\rangle \cdot \left\langle \mathbf {v} _ {j, r} ^ {(t)}, \mathbf {x} \right\rangle \\ \geq \sum_ {r} \left\langle \mathbf {w} _ {j, r} ^ {(0)}, \frac {\mathbf {v} _ {j , 1} ^ {(t)}}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2}} \right\rangle^ {2} + \sum_ {r} \left\langle \mathbf {v} _ {j, r} ^ {(t)}, \frac {\mathbf {v} _ {j , 1} ^ {(t)}}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2}} \right\rangle^ {2} \\ + \sum_ {r} \left\langle \mathbf {w} _ {j, r} ^ {(0)}, \frac {\mathbf {v} _ {j , 1} ^ {(t)}}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2}} \right\rangle \cdot \left\langle \mathbf {v} _ {j, r} ^ {(t)}, \frac {\mathbf {v} _ {j , 1} ^ {(t)}}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2}} \right\rangle \\ \geq \sum_ {r} \left\langle \mathbf {v} _ {j, r} ^ {(t)}, \frac {\mathbf {v} _ {j , 1} ^ {(t)}}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2}} \right\rangle^ {2} - \sum_ {r} \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} ^ {2} - \sum_ {r} \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} \| \mathbf {v} _ {j, r} ^ {(t)} \| _ {2} \\ \geq m \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} ^ {2} + 2 \sum_ {r} \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} \cdot \left\langle \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)}, \frac {\mathbf {v} _ {j , 1} ^ {(t)}}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2}} \right\rangle + \sum_ {r} \left\langle \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)}, \frac {\mathbf {v} _ {j , 1} ^ {(t)}}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2}} \right\rangle^ {2} \\ - \sum_ {r} \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} ^ {2} - \sum_ {r} \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} \left(\| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} + \| \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2}\right) \\ \geq m \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} ^ {2} - \left(\sum_ {r} 2 \| \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} + \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2}\right) \cdot \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} \\ - \left(\sum_ {r} \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} ^ {2} + \| \mathbf {w} _ {j, r} ^ {(0)} \| _ {2} \| \mathbf {v} _ {j, r} ^ {(t)} - \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2}\right) \\ \geq m \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} ^ {2} - m C _ {3} \| \mathbf {v} _ {j, 1} ^ {(t)} \| _ {2} - m C _ {4} \\ \end{array}
$$
where the first inequality is by taking  $\mathbf{x} = \mathbf{v}_{j,1}^{(t)} / \| \mathbf{v}_{j,1}^{(t)}\|_2$ ; the second inequality is by Cauchy inequality; the third inequality by breaking  $\mathbf{v}_{j,r}^{(t)}$  down into  $\mathbf{v}_{j,1}^{(t)} + \mathbf{v}_{j,r}^{(t)} - \mathbf{v}_{j,1}^{(t)}$  and then expanding the
first term as well as applying triangle inequality to the last term; the fourth inequality is by Cauchy inequality; the last inequality is by Lemma D.1, Lemma D.2 and taking
$$
C _ {3} = 1. 5 \sigma_ {0} \sqrt {d} + 3 C ^ {\prime} \sqrt {n} R _ {\mathrm {m i n}} ^ {- 1},
$$
$$
C _ {4} = 1. 5 \sigma_ {0} ^ {2} d + 3 C ^ {\prime} \sigma_ {0} \sqrt {d} \sqrt {n} R _ {\mathrm {m i n}} ^ {- 1}.
$$
By leverage the upper bound of  $\| \mathbf{W}_j^{(t)}\| _F^2$  as well as the lower bound of  $\| \mathbf{W}_j^{(t)}\| _2^2$ , we can get
$$
\frac {\| \mathbf {W} _ {j} ^ {(t)} \| _ {F} ^ {2}}{\| \mathbf {W} _ {j} ^ {(t)} \| _ {2} ^ {2}} \leq \frac {\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2} ^ {2} + C _ {1} \| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2} + C _ {2}}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2} ^ {2} - C _ {3} \| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2} - C _ {4}}.
$$
Since  $\| \mathbf{W}_j^{(t)}\| _F^2 /\| \mathbf{W}_j^{(t)}\| _2^2\geq 1$ $\| \mathbf{v}_{j,1}^{(t)}\|_{2} = \Theta (\log t)$  and  $C_1,C_2,C_3,C_4$  are constants, it follows that
$$
\lim _ {t \to \infty} \frac {\| \mathbf {W} _ {j} ^ {(t)} \| _ {F} ^ {2}}{\| \mathbf {W} _ {j} ^ {(t)} \| _ {2} ^ {2}} = 1,
$$
$$
\begin{array}{l} \frac {\| \mathbf {W} _ {j} ^ {(t)} \| _ {F} ^ {2}}{\| \mathbf {W} _ {j} ^ {(t)} \| _ {2} ^ {2}} - 1 \leq \frac {\left(C _ {1} + C _ {3}\right) \| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2} + \left(C _ {2} + C _ {4}\right)}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2} ^ {2} - C _ {3} \| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2} - C _ {4}} \\ \preceq \frac {C _ {1} + C _ {3}}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2}} \\ \leq \frac {6 (C ^ {\prime} \sqrt {n} R _ {\operatorname* {m i n}} ^ {- 1} + \sigma_ {0} \sqrt {d})}{\| \mathbf {v} _ {j , 1} ^ {(t)} \| _ {2}} \\ = \Theta \Big (\frac {\sqrt {n} R _ {\mathrm {m i n}} ^ {- 1} + \sigma_ {0} \sqrt {d}}{\sqrt {n} R _ {\mathrm {m a x}} ^ {- 1} \log (t)} \Big) = \Theta \Big (\frac {1 + \sigma_ {0} \sqrt {d / n} R _ {\mathrm {m a x}}}{\log (t)} \Big), \\ \end{array}
$$
E Coefficient Analysis of ReLU
In this section, we discuss the stable rank of two-layer ReLU neural network, which is defined as
$$
f (\mathbf {W}, \mathbf {x}) = F _ {+ 1} (\mathbf {W} _ {+ 1}, \mathbf {x}) - F _ {+ 1} (\mathbf {W} _ {+ 1}, \mathbf {x}),
$$
$$
F _ {j} \left(\mathbf {W} _ {j}, \mathbf {x}\right) = \frac {1}{m} \sum_ {r = 1} ^ {m} \sigma \left(\left\langle \mathbf {w} _ {j, r}, \mathbf {x} \right\rangle\right), \tag {E.1}
$$
where  $\sigma (z) = \max \{0,z\}$  is ReLU activation function.
These results are based on the conclusions in Section B, which hold with high probability. Denote by  $\mathcal{E}_{\mathrm{prelim}}^{\prime}$  the event that all the results in Section B hold (for a given  $\delta$ , we see  $\mathbb{P}(\mathcal{E}_{\mathrm{prelim}}^{\prime}) \geq 1 - 2\delta$  by a union bound). For simplicity and clarity, we state all the results in this and the following sections conditional on  $\mathcal{E}_{\mathrm{prelim}}^{\prime}$ .
Denote  $\beta = \max_{i,j,r}\{|\langle \mathbf{w}_{j,r}^{(0)},\mathbf{x}_i\rangle |\}$ ,  $R_{\mathrm{max}} = \max_{i\in [n]}\| \mathbf{x}_i\| _2$ ,  $R_{\mathrm{min}} = \min_{i\in [n]}\| \mathbf{x}_i\| _2$ ,  $p = \max_{i\neq k}|\langle \mathbf{x}_i,\mathbf{x}_k\rangle |$  and suppose  $R = R_{\mathrm{max}} / R_{\mathrm{min}}$  is at most an absolute constant. Here we list the exact conditions for  $\eta, \sigma_0, R_{\mathrm{min}}, R_{\mathrm{max}}, p$  required by the proofs in this section:
$$
\sigma_ {0} \leq \left(C R _ {\max } \sqrt {\log (m n / \delta)}\right) ^ {- 1}, \tag {E.2}
$$
$$
\eta \leq \left(C R _ {\max } ^ {2} / n m\right) ^ {- 1}, \tag {E.3}
$$
$$
R _ {\min } ^ {2} \geq C R ^ {2} n p, \tag {E.4}
$$
$$
\beta \leq c, \tag {E.5}
$$
$$
R _ {\min } ^ {- 2} n p \leq c, \tag {E.6}
$$
$$
R _ {\min } ^ {- 2} R ^ {2} n p \leq c, \tag {E.7}
$$
We first introduce the following lemma which characterizes the increasing rate of coefficients  $\rho_{j,r,i}^{(t)}$ .
Lemma E.1. For two-layer ReLU neural network defined in (E.1), under the same condition as Theorem 4.3, the decomposition coefficients  $\rho_{j,r,i}^{(t)}$  satisfy following properties:
-  $\overline{\rho}_{y_i,r,i}^{(t)}\geq c_1|\rho_{j,r',i'}^{(t)}|$  for any  $r\in S_i^{(0)},r'\in [m],j\in \{\pm 1\}$  and  $i,i'\in [n]$
-  $\overline{\rho}_{y_i,r,i}^{(t)}\geq c_2\log \left(1 + \frac{\eta|S_i^{(0)}||\mathbf{x}_i||_2^2e^{-\beta}}{2nm^2}\cdot t\right)$  for any  $r\in S_i^{(0)}$  and  $i\in [n]$ ,
-  $\overline{\rho}_{y_i,r,i}^{(t)}\leq c_3\log \left(1 + \frac{2\eta|S_i^{(0)}||\mathbf{x}_i||_2^2e^{2\beta}}{nm^2}\cdot t\right)$  for any  $r\in S_i^{(0)}$  and  $i\in [n]$ ,
where  $c_{1}, c_{2}, c_{3}$  are constants. And the following activation pattern is also observed:  $S_{i}^{(0)} \subseteq S_{i}^{(t)}$  where  $S_{i}^{(t)} := \{r \in [m] : \langle \mathbf{w}_{y_{i}, r}^{(t)}, \mathbf{x}_{i} \rangle \geq 0\}$ , i.e., the on-diagonal neuron activated at initialization will remain activated throughout the training.
Proof of Lemma E.1. We first show that the first bullet and  $S_{i}^{(0)} \subseteq S_{i}^{(t)}$  hold for  $t \leq T_1 = C\eta^{-1}nmR_{\max}^{-2}$  where  $C = \Theta (1)$  is a constant. Now we prove this by induction. When  $t = 0$ , the two hypotheses hold naturally. Suppose that there exists time  $\widetilde{t} \leq T_1$  such that the two hypotheses hold for all time  $t \leq \widetilde{t} - 1$ . We aim to prove they also hold for  $t = \widetilde{t}$ . Recall from Lemma B.3 that
$$
\bar {\rho} _ {j, r, i} ^ {(t + 1)} \leq \bar {\rho} _ {j, r, i} ^ {(t)} + \frac {\eta}{n m} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \leq \bar {\rho} _ {j, r, i} ^ {(t)} + \frac {\eta R _ {\max } ^ {2}}{n m}, \tag {E.8}
$$
$$
\left| \underline {{\rho}} _ {j, r, i} ^ {(t + 1)} \right| \leq \left| \underline {{\rho}} _ {j, r, i} ^ {(t)} \right| + \frac {\eta}{n m} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \leq \left| \underline {{\rho}} _ {j, r, i} ^ {(t)} \right| + \frac {\eta R _ {\operatorname* {m a x}} ^ {2}}{n m}. \tag {E.9}
$$
Therefore,  $\max_{j,r,i}\{\overline{\rho}_{j,r,i}^{(t)},|\underline{\rho}_{j,r,i}^{(t)}|\} = O(1)$  for any  $t\leq T_1$  and hence  $\max_i\{F_{+1}(\mathbf{W}_{+1}^{(t)},\mathbf{x}_i),F_{-1}(\mathbf{W}_{-1}^{(t)},\mathbf{x}_i)\} = O(1)$  for any  $t\leq T_1$ . Thus there exists a positive constant  $c$  such that  $|\ell_i^{\prime (t)}|\geq c$  for any  $t\leq T_1$ . By induction hypothesis, we have  $S_{i}^{(0)}\subseteq S_{i}^{(t)}$  for all  $0\le t\le \widetilde{t} -1$  and hence  $\sigma^{\prime}(\langle \mathbf{w}_{y_i,r}^{(t)},\mathbf{x}_i\rangle) = 1$  for all  $0\leq t\leq \widetilde{t} -1$ . And it follows that for  $r\in S_i^{(0)}$
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t + 1)} = \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} + \frac {\eta}{n m} \cdot | \ell_ {i} ^ {\prime (t)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \geq \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} + \frac {c \eta}{n m} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \forall 0 \leq t \leq \widetilde {t} - 1,
$$
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(\widetilde {t})} \geq \frac {c \eta \widetilde {t}}{n m} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \geq \frac {c \eta R _ {\min } ^ {2} \widetilde {t}}{n m}. \tag {E.10}
$$
On the other hand, by (E.8) and (E.9), we have
$$
\bar {\rho} _ {j, r ^ {\prime}, i ^ {\prime}} ^ {(\widetilde {t})} \leq \frac {\eta R _ {\max } ^ {2} \widetilde {t}}{n m}, | \underline {{\rho}} _ {j, r ^ {\prime}, i ^ {\prime}} ^ {(\widetilde {t})} | \leq \frac {\eta R _ {\max } ^ {2} \widetilde {t}}{n m} \Longrightarrow | \rho_ {j, r ^ {\prime}, i ^ {\prime}} ^ {(\widetilde {t})} | \leq \frac {\eta R _ {\max } ^ {2} \widetilde {t}}{n m}. \tag {E.11}
$$
Dividing (E.10) by (E.11), we can get
$$
\frac {\bar {\rho} _ {y _ {i} , r , i} ^ {(\tilde {t})}}{| \rho_ {j , r ^ {\prime} , i ^ {\prime}} ^ {(\tilde {t})} |} \geq \frac {c R _ {\operatorname* {m i n}} ^ {2}}{R _ {\operatorname* {m a x}} ^ {2}}, \forall r \in S _ {i} ^ {(0)}, j \in \{\pm 1 \}, i, i ^ {\prime} \in [ n ], \tag {E.12}
$$
which indicates that the first bullet holds for time  $t = \widetilde{t}$  as long as  $c_{1} \leq (cR_{\mathrm{min}}^{2}) / R_{\mathrm{max}}^{2}$ . For  $r \in S_i^{(0)}$ , we have
$$
\begin{array}{l} \langle \mathbf {w} _ {y _ {i}, r} ^ {(\widetilde {t})}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \sum_ {i ^ {\prime} = 1} ^ {n} \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t})} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle \\ = \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \right\rangle + \overline {{\rho}} _ {y _ {i}, r, i} ^ {(\widetilde {t})} + \sum_ {i ^ {\prime} \neq i} \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t})} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \left\langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \right\rangle \\ \geq \overline {{\rho}} _ {y _ {i}, r, i} ^ {(\widetilde {t})} - \sum_ {i ^ {\prime} \neq i} | \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t})} | R _ {\min } ^ {- 2} p \\ \geq \bar {\rho} _ {y _ {i}, r, i} ^ {(\tilde {t})} - \sum_ {i ^ {\prime} \neq i} \frac {R _ {\max} ^ {2}}{c R _ {\min} ^ {2}} \bar {\rho} _ {y _ {i}, r, i} ^ {(\tilde {t})} \cdot R _ {\min } ^ {- 2} p \\ \geq \left(1 - \frac {R _ {\operatorname* {m a x}} ^ {2}}{c R _ {\operatorname* {m i n}} ^ {4}} p n\right) \cdot \bar {\rho} _ {y _ {i}, r, i} ^ {(\tilde {t})} \geq 0, \\ \end{array}
$$
where the second inequality is by (E.12). This implies that  $S_{i}^{(0)} \subseteq S_{i}^{(t)}$  holds for time  $t = \widetilde{t}$ , which completes the induction. By then, we have already proved that the first bullet and  $S_{i}^{(0)} \subseteq S_{i}^{(t)}$  hold for  $t \leq T_1 = C\eta^{-1}nmR_{\max}^{-2}$ .
Next, we will prove by induction that the three bullets as well as  $S_{i}^{(0)} \subseteq S_{i}^{(t)}$  hold for any time  $t \geq 0$ . The second and third bullets are obvious at  $t = 0$  as all the coefficients are zero. Suppose there exists  $\widetilde{t}$  such that the three bullets as well as  $S_{i}^{(0)} \subseteq S_{i}^{(t)}$  hold for all time  $0 \leq t \leq \widetilde{t} - 1$ . We aim to prove that they also hold for  $t = \widetilde{t}$ . We first prove that the second and third bullets hold for  $t = \widetilde{t}$ . To prove this, we first provide more precise upper and lower bounds for  $|\ell_{i}^{\prime (t)}|$ . For lower bound, we have
$$
\begin{array}{l} | \ell_ {i} ^ {(t)} | = \frac {1}{1 + \exp \left\{F _ {y _ {i}} \left(\mathbf {W} _ {y _ {i}} ^ {(t)} , \mathbf {x} _ {i}\right) - F _ {- y _ {i}} \left(\mathbf {W} _ {- y _ {i}} ^ {(t)} , \mathbf {x} _ {i}\right) \right\}} \\ \geq \frac {1}{1 + \exp \left\{F _ {y _ {i}} \left(\mathbf {W} _ {y _ {i}} ^ {(t)} , \mathbf {x} _ {i}\right) \right\}} \\ = \frac {1}{1 + \exp \left\{\frac {1}{m} \sum_ {r \in S _ {i} ^ {(t)}} \langle \mathbf {w} _ {y _ {i} , r} ^ {(t)}, \mathbf {x} _ {i} \rangle \right\}} \tag {E.13} \\ \end{array}
$$
$$
\begin{array}{l} \sum_ {r \in S _ {i} ^ {(t)}} \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \right\rangle = \sum_ {r \in S _ {i} ^ {(t)}} \left(\left\langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \right\rangle + \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + \sum_ {i ^ {\prime} \neq i} \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(t)} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \left\langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \right\rangle\right) \\ \leq \sum_ {r \in S _ {i} ^ {(t)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} + \sum_ {r \in S _ {i} ^ {(t)}} \sum_ {i ^ {\prime} \neq i} | \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(t)} | R _ {\min } ^ {- 2} p + | S _ {i} ^ {(t)} | \cdot \beta \\ \leq \sum_ {r \in S _ {i} ^ {(t)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + \frac {| S _ {i} ^ {(t)} |}{c _ {1} | S _ {i} ^ {(0)} |} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i ^ {\prime}} ^ {(t)} R _ {\min } ^ {- 2} p n + | S _ {i} ^ {(t)} | \cdot \beta \\ \leq \frac {\left| S _ {i} ^ {(t)} \right|}{\left| S _ {i} ^ {(0)} \right|} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} + \frac {\left| S _ {i} ^ {(t)} \right|}{c _ {1} \left| S _ {i} ^ {(0)} \right|} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(t)} R _ {\min } ^ {- 2} p n + \left| S _ {i} ^ {(t)} \right| \cdot \beta \\ \leq c ^ {\prime} \left(1 + R _ {\min } ^ {- 2} p n / c _ {1}\right) \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + | S _ {i} ^ {(t)} | \cdot \beta , \tag {E.14} \\ \end{array}
$$
where the first inequality is by triangle inequality; the second inequality is by the first induction hypothesis that  $\overline{\rho}_{y_i,r,i}^{(t)}\geq c_1|\rho_{y_i,r',i'}^{(t)}|$  for  $r\in S_i^{(0)}$  and  $0\leq t\leq \widetilde{t} -1$  and hence  $|\rho_{y_i,r',i'}^{(t)}|\leq \frac{1}{c_1|S_i^{(0)}|}\sum_{r\in S_i^{(0)}}\overline{\rho}_{y_i,r,i}^{(t)}$ ; the third inequality is by
$$
\overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} = \frac {\eta}{n m} \sum_ {s = 0} ^ {t - 1} | \ell_ {i} ^ {\prime (s)} | \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {y _ {i}, r ^ {\prime}} ^ {(s)}, \mathbf {x} _ {i} \rangle) \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \leq \frac {\eta}{n m} \sum_ {s = 0} ^ {t - 1} | \ell_ {i} ^ {\prime (s)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2},
$$
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} = \frac {\eta}{n m} \sum_ {s = 0} ^ {t - 1} | \ell_ {i} ^ {\prime (s)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2},
$$
and hence  $\overline{\rho}_{y_i,r',i}^{(t)}\leq \overline{\rho}_{y_i,r,i}^{(t)},\forall r'\in S_i^{(t)}\setminus S_i^{(0)},r\in S_i^{(0)}$  for  $0\leq t\leq \widetilde{t} -1$ ; the last inequality is by  $|S_i^{(t)}|\leq m\leq c'|S_i^{(0)}|$  and  $c^\prime$  can be taken as 2.5 by Lemma B.2. By plugging (E.14) back into (E.13), we can get
$$
\begin{array}{l} | \ell_ {i} ^ {(t)} | \geq \frac {1}{1 + \exp \left\{\frac {c ^ {\prime} (1 + R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1})}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i} , r , i} ^ {(t)} + \frac {| S _ {i} ^ {(t)} |}{m} \cdot \beta \right\}} \\ \geq \frac {1}{1 + \exp \left\{\frac {c ^ {\prime} \left(1 + R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1}\right)}{m} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i} , r , i} ^ {(t)} + \beta \right\}} \tag {E.15} \\ \geq \frac {1}{2} \exp \Bigg \{- \frac {c ^ {\prime} (1 + R _ {\min} ^ {- 2} p n / c _ {1})}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \beta \Bigg \}, \forall 0 \leq t \leq \widetilde {t} - 1. \\ \end{array}
$$
For upper bound of  $|\ell_i^{\prime (t)}|$ , we first bound  $F_{y_i}(\mathbf{W}_{y_i}^{(t)},\mathbf{x}_i) - F_{-y_i}(\mathbf{W}_{-y_i}^{(t)},\mathbf{x}_i)$  as follows: