SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} F _ {y _ {i}} (\mathbf {W} _ {y _ {i}} ^ {(t)}, \mathbf {x} _ {i}) - F _ {- y _ {i}} (\mathbf {W} _ {- y _ {i}} ^ {(t)}, \mathbf {x} _ {i}) \\ \geq \frac {1}{m} \left(\sum_ {r \in S _ {i} ^ {(t)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} - \sum_ {r \in S _ {i} ^ {(t)}} \sum_ {i ^ {\prime} \neq i} | \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(t)} | R _ {\min } ^ {- 2} p - | S _ {i} ^ {(t)} | \cdot \beta\right) - \frac {1}{m} \sum_ {r = 1} ^ {m} \left(\beta + \sum_ {i ^ {\prime} \neq i} | \rho_ {- y _ {i}, r, i ^ {\prime}} ^ {(t)} | R _ {\min } ^ {- 2} p\right) \\ \geq \frac {1}{m} \sum_ {r \in S _ {i} ^ {(t)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \frac {| S _ {i} ^ {(t)} |}{c _ {1} m | S _ {i} ^ {(0)} |} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} R _ {\mathrm {m i n}} ^ {- 2} p n - \frac {1}{c _ {1} | S _ {i} ^ {(0)} |} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} R _ {\mathrm {m i n}} ^ {- 2} p n - 2 \beta \\ \geq \frac {1}{m} \sum_ {r \in S _ {i} ^ {(t)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \frac {2}{c _ {1} | S _ {i} ^ {(0)} |} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} R _ {\mathrm {m i n}} ^ {- 2} p n - 2 \beta \\ \geq \frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \frac {2 c ^ {\prime} R _ {\mathrm {m i n}} ^ {- 2} p n}{c _ {1} m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - 2 \beta \\ = \frac {1 - 2 c ^ {\prime} R _ {\mathrm {m i n}} ^ {- 2} p n / c _ {1}}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - 2 \beta , \\ \end{array}
$$
where the first inequality is by
$$
\begin{array}{l} F _ {y _ {i}} \left(\mathbf {W} _ {y _ {i}} ^ {(t)}, \mathbf {x} _ {i}\right) = \frac {1}{m} \sum_ {r \in S _ {i} ^ {(t)}} \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \right\rangle \\ = \frac {1}{m} \sum_ {r \in S _ {i} ^ {(t)}} \left(\langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} + \sum_ {i ^ {\prime} \neq i} \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(t)} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle\right) \\ \geq \frac {1}{m} \bigg (\sum_ {r \in S _ {i} ^ {(t)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \sum_ {r \in S _ {i} ^ {(t)}} \sum_ {i ^ {\prime} \neq i} | \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(t)} | R _ {\min} ^ {- 2} p - | S _ {i} ^ {(t)} | \cdot \beta \bigg), \\ \end{array}
$$
$$
\begin{array}{l} \langle \mathbf {w} _ {- y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {- y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \sum_ {i ^ {\prime} = 1} ^ {n} \rho_ {- y _ {i}, r, i ^ {\prime}} ^ {(t)} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle \\ = \langle \mathbf {w} _ {- y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} + \sum_ {i ^ {\prime} \neq i} \rho_ {- y _ {i}, r, i ^ {\prime}} ^ {(t)} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle \\ \leq \beta + \sum_ {i ^ {\prime} \neq i} | \rho_ {- y _ {i}, r, i ^ {\prime}} ^ {(t)} | R _ {\mathrm {m i n}} ^ {- 2} p, \\ \end{array}
$$
$$
F _ {- y _ {i}} \left(\mathbf {W} _ {- y _ {i}} ^ {(t)}, \mathbf {x} _ {i}\right) = \frac {1}{m} \sum_ {r = 1} ^ {m} \sigma \left(\left\langle \mathbf {w} _ {- y _ {i}, r} ^ {(t)}, \mathbf {x} _ {i} \right\rangle\right) \leq \frac {1}{m} \sum_ {r = 1} ^ {m} \left(\beta + \sum_ {i ^ {\prime} \neq i} | \rho_ {- y _ {i}, r, i ^ {\prime}} ^ {(t)} | R _ {\min } ^ {- 2} p\right); \tag {E.16}
$$
the second inequality is by the first induction hypothesis that  $\overline{\rho}_{y_i,r,i}^{(t)}\geq c_1|\rho_{y_i,r',i'}^{(t)}|,\overline{\rho}_{y_i,r,i}^{(t)}\geq$ $c_{1}|\rho_{-y_{i},r^{\prime},i^{\prime}}^{(t)}|$  and hence  $|\rho_{y_i,r',i'}^{(t)}|\leq \frac{1}{c_1|S_i^{(0)}|}\sum_{r\in S_i^{(0)}}\overline{\rho}_{y_i,r,i}^{(t)},|\rho_{-y_i,r',i'}^{(t)}|\leq \frac{1}{c_1|S_i^{(0)}|}\sum_{r\in S_i^{(0)}}\overline{\rho}_{y_i,r,i}^{(t)}$  for  $r\in S_i^{(0)}$  and  $0\leq t\leq \widetilde{t} -1$ ; the third inequality is by  $|S_i^{(t)}|\leq m$ ; the fourth inequality is by  $m\leq c'|S_i^{(0)}|$ . Therefore,
$$
\begin{array}{l} | \ell_ {i} ^ {\prime (t)} | = \frac {1}{1 + \exp \left\{F _ {y _ {i}} (\mathbf {W} _ {y _ {i}} ^ {(t)} , \mathbf {x} _ {i}) - F _ {- y _ {i}} (\mathbf {W} _ {- y _ {i}} ^ {(t)} , \mathbf {x} _ {i}) \right\}} \\ \leq \exp \left\{- F _ {y _ {i}} \left(\mathbf {W} _ {y _ {i}} ^ {(t)}, \mathbf {x} _ {i}\right) + F _ {- y _ {i}} \left(\mathbf {W} _ {- y _ {i}} ^ {(t)}, \mathbf {x} _ {i}\right) \right\} \\ \leq \exp \Bigg \{- \frac {1 - 2 c ^ {\prime} R _ {\mathrm {m i n}} ^ {- 2} p n / c _ {1}}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} + 2 \beta \Bigg \}, \forall 0 \leq t \leq \widetilde {t} - 1. \\ \end{array}
$$
By the induction hypothesis, we know that  $S_{i}^{(0)} \subseteq S_{i}^{(t)}$  for all  $0 \leq t \leq \widetilde{t} - 1$  and hence  $\sigma'(\langle \mathbf{w}_{y_i,r}^{(t)}, \mathbf{x}_i \rangle) = 1$  for all  $r \in S_{i}^{(0)}$  and  $0 \leq t \leq \widetilde{t} - 1$ . By (E.15) and (E.16), it follows that for all  $0 \leq t \leq \widetilde{t} - 1$
$$
\sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t + 1)} \geq \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + \frac {\eta \left| S _ {i} ^ {(0)} \right| \left\| \mathbf {x} _ {i} \right\| _ {2} ^ {2} e ^ {- \beta}}{2 n m} \cdot \exp \left\{- \frac {c ^ {\prime} \left(1 + R _ {\min } ^ {- 2} p n / c _ {1}\right)}{m} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} \right\}, \tag {E.17}
$$
$$
\sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t + 1)} \leq \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + \frac {\eta \left| S _ {i} ^ {(0)} \right| \left\| \mathbf {x} _ {i} \right\| _ {2} ^ {2} e ^ {2 \beta}}{n m} \cdot \exp \left\{- \frac {1 - 2 c ^ {\prime} R _ {\min } ^ {- 2} p n / c _ {1}}{m} \sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} \right\}. \tag {E.18}
$$
By applying Lemma H.2 to (E.17) and taking  $x_{t} = \frac{c^{\prime}(1 + R_{\min}^{-2}pn / c_{1})}{m}\sum_{r\in S_{i}^{(0)}}\overline{\rho}_{y_{i},r,i}^{(t)}$ , we can get
$$
\sum_ {r \in S _ {i} ^ {(0)}} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} \geq \frac {m}{c ^ {\prime} \left(1 + R _ {\min } ^ {- 2} p n / c _ {1}\right)} \log \left(1 + \frac {c ^ {\prime} \left(1 + R _ {\min } ^ {- 2} p n / c _ {1}\right) \eta \left| S _ {i} ^ {(0)} \right| \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {- \beta}}{2 n m ^ {2}} \cdot t\right), \forall 0 \leq t \leq \widetilde {t}. \tag {E.19}
$$
By applying Lemma H.1 to (E.18) and taking  $x_{t} = \frac{1 - 2c^{\prime}R_{\mathrm{min}}^{-2}pn / c_{1}}{m}\sum_{r\in S_{i}^{(0)}}\overline{\rho}_{y_{i},r,i}^{(t)}$ , we can get for any  $0\leq t\leq \widetilde{t}$  that
$$
\begin{array}{l} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} \leq \frac {m}{1 - 2 c ^ {\prime} R _ {\mathrm {m i n}} ^ {- 2} p n / c _ {1}} \log \left(1 + \frac {(1 - 2 c ^ {\prime} R _ {\mathrm {m i n}} ^ {- 2} p n / c _ {1}) \eta | S _ {i} ^ {(0)} | \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {2 \beta}}{n m ^ {2}}\right). \\ \exp \left(\frac {\left(1 - 2 c ^ {\prime} R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1}\right) \eta | S _ {i} ^ {(0)} | \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {2 \beta}}{n m ^ {2}}\right) \cdot t\left. \right) \\ \leq \frac {m}{1 - 2 c ^ {\prime} R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1}} \log \left(1 + \frac {2 \left(1 - 2 c ^ {\prime} R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1}\right) \eta \left| S _ {i} ^ {(0)} \right| \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {2 \beta}}{n m ^ {2}} \cdot t\right), \tag {E.20} \\ \end{array}
$$
where the last inequality is by  $\eta \leq (CR_{\mathrm{max}}^2 /nm)^{-1}$ ,  $C$  is a large enough constant and hence  $(1 - 2c'R_{\mathrm{min}}^{-2}pn / c_1)\eta |S_i^{(0)}||\mathbf{x}_i||_2^2 e^{2\beta} / nm^2\leq \log 2$ . Since  $S_{i}^{(0)}\subseteq S_{i}^{(t)}$  for all  $0\leq t\leq \widetilde{t} -1$  and hence  $\sigma^{\prime}(\langle \mathbf{w}_{y_i,r}^{(t)},\mathbf{x}_i\rangle) = 1$  for all  $r\in S_i^{(0)}$  and  $0\leq t\leq \widetilde{t} -1$ , we have
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(t)} = \frac {\eta}{n m} \sum_ {s = 0} ^ {t - 1} \left| \ell_ {i} ^ {\prime (s)} \right| \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \forall 0 \leq t \leq \widetilde {t}.
$$
Accordingly, it holds that
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(t)} = \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)}, \forall r, r ^ {\prime} \in S _ {i} ^ {(0)}, \forall 0 \leq t \leq \widetilde {t}.
$$
Applying this to (E.19) and (E.20), we can get
$$
\begin{array}{l} \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} \geq \frac {m}{c ^ {\prime} \left(1 + R _ {\min } ^ {- 2} p n / c _ {1}\right) \left| S _ {i} ^ {(0)} \right|} \log \left(1 + \frac {c ^ {\prime} \left(1 + R _ {\min } ^ {- 2} p n / c _ {1}\right) \eta \left| S _ {i} ^ {(0)} \right| \left\| \mathbf {x} _ {i} \right\| _ {2} ^ {2} e ^ {- \beta}}{2 n m ^ {2}} \cdot t\right) \\ \geq \frac {1}{c ^ {\prime} (1 + R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1})} \log \left(1 + \frac {c ^ {\prime} (1 + R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1}) \eta | S _ {i} ^ {(0)} | \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {- \beta}}{2 n m ^ {2}} \cdot t\right), \forall 0 \leq t \leq \widetilde {t}, \\ \end{array}
$$
$$
\begin{array}{l} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} \leq \frac {m}{(1 - 2 c ^ {\prime} R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1}) | S _ {i} ^ {(0)} |} \log \left(1 + \frac {2 (1 - 2 c ^ {\prime} R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1}) \eta | S _ {i} ^ {(0)} | \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {2 \beta}}{n m ^ {2}} \cdot t\right), \\ \leq \frac {c ^ {\prime}}{\left(1 - 2 c ^ {\prime} R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1}\right)} \log \left(1 + \frac {2 \left(1 - 2 c ^ {\prime} R _ {\operatorname* {m i n}} ^ {- 2} p n / c _ {1}\right) \eta \left| S _ {i} ^ {(0)} \right| \left\| \mathbf {x} _ {i} \right\| _ {2} ^ {2} e ^ {2 \beta}}{n m ^ {2}} \cdot t\right), \forall 0 \leq t \leq \widetilde {t}, \tag {E.21} \\ \end{array}
$$
$$
c _ {2} = \frac {1}{c ^ {\prime} \left(1 + R _ {\mathrm {m i n}} ^ {- 2} p n / c _ {1}\right)}, c _ {3} = \frac {c ^ {\prime}}{\left(1 - 2 c ^ {\prime} R _ {\mathrm {m i n}} ^ {- 2} p n / c _ {1}\right)},
$$
the above inequalities indicate that the second and third bullets hold at time  $t = \widetilde{t}$ . For the first bullet, it is only necessary to consider the situation where  $\widetilde{t} \geq T_1 = C\eta^{-1}nmR_{\max}^{-2}$ . In order to apply Lemma H.4 (requiring  $b > a$ ), we loosen the bounds in (E.21) as follows:
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(t)} \geq c _ {2} \log \left(1 + \frac {\eta R _ {\min } ^ {2} e ^ {- \beta}}{5 n m} \cdot t\right), \forall 0 \leq t \leq \widetilde {t}, \tag {E.22}
$$
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(t)} \leq c _ {3} \log \left(1 + \frac {6 \eta R _ {\max } ^ {2} e ^ {2 \beta}}{5 n m} \cdot t\right), \forall 0 \leq t \leq \widetilde {t}, \tag {E.23}
$$
where we use  $0.4m \leq |S_i^{(0)}| \leq 0.6m$ .
By applying Lemma H.4 to (E.22) and (E.23), we can get for any  $i, i' \in [n]$ ,  $r \in S_i^{(0)}$ ,  $r' \in S_{i'}^{(0)}$  and  $T_1 \leq t \leq \widetilde{t}$  that
$$
\begin{array}{l} \frac {\overline {{\rho}} _ {y _ {i} , r , i} ^ {(t)}}{\overline {{\rho}} _ {y _ {i ^ {\prime}} , r ^ {\prime} , i ^ {\prime}} ^ {(t)}} \geq \frac {c _ {2}}{c _ {3}} \cdot \frac {\log \left(1 + \frac {\eta R _ {\mathrm {m i n}} ^ {2} e ^ {- \beta}}{5 n m} \cdot t\right)}{\log \left(1 + \frac {6 \eta R _ {\mathrm {m a x}} ^ {2} e ^ {2 \beta}}{5 n m} \cdot t\right)} \\ \geq \frac {c _ {2}}{c _ {3}} \cdot \frac {\log \left(1 + \frac {\eta R _ {\operatorname* {m a x}} ^ {2} e ^ {2 \beta}}{5 n m} \cdot T _ {1}\right)}{\log \left(1 + \frac {6 \eta R _ {\operatorname* {m a x}} ^ {2} e ^ {2 \beta}}{5 n m} \cdot T _ {1}\right)} \\ = \frac {c _ {2}}{c _ {3}} \cdot \frac {\log \left(1 + 0 . 2 C e ^ {- \beta} R _ {\operatorname* {m i n}} ^ {2}\right)}{\log \left(1 + 1 . 2 C e ^ {2 \beta} R _ {\operatorname* {m a x}} ^ {2}\right)}. \\ \end{array}
$$
Notice that  $S_{i'}^{(0)} \subseteq S_{i'}^{(t)}$  for all  $0 \leq t \leq \widetilde{t} - 1$  and hence  $\sigma'(\langle \mathbf{w}_{y_{i'}, r}', \mathbf{x}_{i'} \rangle) = 1$  for all  $r \in S_{i'}^{(0)}$  and  $0 \leq t \leq \widetilde{t} - 1$ , we have
$$
\begin{array}{l} | \rho_ {j, r ^ {\prime \prime}, i ^ {\prime}} ^ {(t)} | = \frac {\eta}{n m} \sum_ {s = 0} ^ {t - 1} | \ell_ {i ^ {\prime}} ^ {\prime (s)} | \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {j, r ^ {\prime \prime}} ^ {(s)}, \mathbf {x} _ {i ^ {\prime}} \rangle) \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \\ \leq \frac {\eta}{n m} \sum_ {s = 0} ^ {t - 1} \left| \ell_ {i ^ {\prime}} ^ {\prime (s)} \right| \cdot \left\| \mathbf {x} _ {i ^ {\prime}} \right\| _ {2} ^ {2}, \quad \forall j \in \{\pm 1 \}, r ^ {\prime \prime} \in [ m ], i ^ {\prime} \in [ n ], \\ \end{array}
$$
$$
\overline {{\rho}} _ {y _ {i ^ {\prime}}, r ^ {\prime}, i ^ {\prime}} ^ {(t)} = \frac {\eta}{n m} \sum_ {s = 0} ^ {t - 1} | \ell_ {i ^ {\prime}} ^ {\prime (s)} | \cdot \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {2}, \quad \forall r ^ {\prime} \in S _ {i ^ {\prime}} ^ {(0)}, i ^ {\prime} \in [ n ],
$$
and hence  $|\rho_{j,r',i'}^{(t)}| \leq \overline{\rho}_{y_{i'},r',i'}^{(t)}$  for  $0 \leq t \leq \widetilde{t}$ . Therefore, as long as
$$
c _ {1} \leq \frac {c _ {2}}{c _ {3}} \cdot \frac {\log \left(1 + 0 . 2 C e ^ {- \beta} R _ {\mathrm {m i n}} ^ {2}\right)}{\log \left(1 + 1 . 2 C e ^ {2 \beta} R _ {\mathrm {m a x}} ^ {2}\right)},
$$
the first bullet hold for time  $t = \widetilde{t}$ . This condition holds as long as
$$
c ^ {\prime} = 2. 5,
$$
$$
2 c ^ {\prime} c _ {1} ^ {- 1} R _ {\min } ^ {- 2} p n \leq 0. 5 \quad \Longrightarrow \quad c _ {2} \geq 0. 3 7, c _ {3} \leq 5,
$$
$$
c _ {1} = \frac {\log \left(1 + 0 . 2 C e ^ {- \beta} R _ {\mathrm {m i n}} ^ {2}\right)}{1 4 \log \left(1 + 1 . 2 C e ^ {2 \beta} R _ {\mathrm {m a x}} ^ {2}\right)}.
$$
Finally, we verify that  $S_{i}^{(0)}\subseteq S_{i}^{(\widetilde{t})}$ . For  $r\in S_i^{(0)}$ , we have
$$
\begin{array}{l} \langle \mathbf {w} _ {y _ {i}, r} ^ {(\widetilde {t})}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \sum_ {i ^ {\prime} = 1} ^ {n} \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t})} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle \\ = \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \right\rangle + \overline {{\rho}} _ {y _ {i}, r, i} ^ {(\bar {t})} + \sum_ {i ^ {\prime} \neq i} \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(\bar {t})} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \left\langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \right\rangle \\ \geq \overline {{\rho}} _ {y _ {i}, r, i} ^ {(\widetilde {t})} - \sum_ {i ^ {\prime} \neq i} | \rho_ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t})} | R _ {\mathrm {m i n}} ^ {- 2} p \\ \geq \bar {\rho} _ {y _ {i}, r, i} ^ {(\tilde {t})} - \left(R _ {\min } ^ {- 2} p n / c _ {1}\right) \bar {\rho} _ {y _ {i}, r, i ^ {\prime}} ^ {(\tilde {t})} \\ = \left(1 - R _ {\min } ^ {- 2} p n / c _ {1}\right) \cdot \overline {{\rho}} _ {y _ {i}, r, i} ^ {(\widetilde {t})} \geq 0, \\ \end{array}
$$
where the second inequality is by  $|\rho_{y_i,r,i'}^{(\widetilde{t})}|\leq \overline{\rho}_{y_i,r,i}^{(\widetilde{t})} / c_1$ . This implies that  $S_{i}^{(0)}\subseteq S_{i}^{(t)}$  holds for time  $t = \widetilde{t}$ , which completes the induction.
Next, we show that  $|\underline{\rho}_{-y_i,r,i'}^{(t)}|$  will be much smaller than  $|\overline{\rho}_{y_i,r',i}^{(t)}|$  as the training goes on.
Lemma E.2. There exists time  $T_{2}$  and constant  $c$  such that for any time  $t \geq T_{2}$
$$
| \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(t)} | \leq c R _ {\min } ^ {- 2} p n | \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} |,
$$
where  $r\in [m],r'\in S_i^{(0)}$  and  $i,i'\in [n]$  satisfying  $-y_{i} = y_{i'}$
Proof of Lemma E.2. First, we will prove by induction that for  $r \in [m]$ ,  $r' \in S_i^{(0)}$  and  $i, i' \in [n]$  satisfying  $-y_i = y_{i'}$  it holds that
$$
\left| \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(t)} \right| \leq \beta + 1 + R _ {\min } ^ {- 2} p n \left| \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} \right| / c _ {1}. \tag {E.24}
$$
This result holds naturally when  $t = 0$  since all the coefficients are zero. Suppose that there exists time  $\widetilde{t}$  such that the induction hypothesis (E.24) holds for all time  $t \leq \widetilde{t} - 1$ . We aim to prove that (E.24) also holds for  $t = \widetilde{t}$ . In the following analysis, two cases will be considered:
$$
| \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t} - 1)} | > \beta + \sum_ {k \neq i ^ {\prime}} | \rho_ {y _ {i}, r, k} ^ {(\widetilde {t} - 1)} | \| \mathbf {x} _ {k} \| _ {2} ^ {- 2} p \mathrm {a n d} | \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t} - 1)} | \leq \beta + \sum_ {k \neq i ^ {\prime}} | \rho_ {y _ {i}, r, k} ^ {(\widetilde {t} - 1)} | \| \mathbf {x} _ {k} \| _ {2} ^ {- 2} p.
$$
For if  $|\underline{\rho}_{y_i,r,i'}^{(\tilde{t} -1)}| > \beta +\sum_{k\neq i'}|\rho_{y_i,r,k}^{(\tilde{t} -1)}||\mathbf{x}_k||_2^{-2}p$  then by the decomposition (5.1) we have
$$
\begin{array}{l} \langle \mathbf {w} _ {y _ {i}, r} ^ {(\widetilde {t} - 1)}, \mathbf {x} _ {i ^ {\prime}} \rangle = \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i ^ {\prime}} \rangle + \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t} - 1)} + \sum_ {k \neq i ^ {\prime}} \rho_ {y _ {i}, r, k} ^ {(\widetilde {t} - 1)} \| \mathbf {x} _ {k} \| _ {2} ^ {- 2} \langle \mathbf {x} _ {k}, \mathbf {x} _ {i ^ {\prime}} \rangle \\ \leq \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t} - 1)} + \beta + \sum_ {k \neq i ^ {\prime}} | \rho_ {y _ {i}, r, k} ^ {(\widetilde {t} - 1)} | \| \mathbf {x} _ {k} \| _ {2} ^ {- 2} p <   0. \\ \end{array}
$$
$$
\underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t})} = \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t} - 1)} + \frac {\eta}{n m} \cdot \ell_ {i} ^ {\prime (\widetilde {t} - 1)} \cdot \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {y _ {i}, r} ^ {(\widetilde {t} - 1)}, \mathbf {x} _ {i ^ {\prime}} \right\rangle\right) \cdot \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {2} = \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t} - 1)}.
$$
Therefore, by induction hypothesis (E.24) at time  $t = \widetilde{t} - 1$ , we have
$$
\underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\tilde {t})} = \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\tilde {t} - 1)} \leq \beta + 1 + R _ {\min} ^ {- 2} p n | \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(\tilde {t} - 1)} | / c _ {1} \leq \beta + 1 + R _ {\min} ^ {- 2} p n | \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(\tilde {t})} | / c _ {1}.
$$
For if  $|\underline{\rho}_{y_i,r,i'}^{(\widetilde{t} -1)}|\leq \beta +\sum_{k\neq i'}|\rho_{y_i,r,k}^{(\widetilde{t} -1)}||\mathbf{x}_k||_2^{-2}p,$  by the first bullet in Lemma E.1, we have
$$
\left| \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t} - 1)} \right| \leq \beta + \sum_ {k \neq i ^ {\prime}} \left| \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(\widetilde {t} - 1)} \right| \| \mathbf {x} _ {k} \| _ {2} ^ {- 2} p / c _ {1} \leq \beta + \left| \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(\widetilde {t} - 1)} \right| R _ {\min } ^ {- 2} p n / c _ {1}, \tag {E.25}
$$
$$
\begin{array}{l} - \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t})} = - \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t} - 1)} + \frac {\eta}{n m} \cdot | \ell_ {i} ^ {\prime} (\widetilde {t} - 1) | \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {y _ {i}, r} ^ {(\widetilde {t} - 1)}, \mathbf {x} _ {i ^ {\prime}} \rangle) \cdot \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {2} \\ \leq - \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(\widetilde {t} - 1)} + \frac {\eta R _ {\max} ^ {2}}{n m} \\ \leq \beta + 1 + | \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} | R _ {\mathrm {m i n}} ^ {- 2} p n / c _ {1}, \\ \end{array}
$$
where the last inequality is by (E.25) and  $\eta \leq (CR_{\max}^2 /nm)^{-1}$  with a sufficiently large constant  $C$ . This demonstrates that inequality (E.24) holds for  $t = \widetilde{t}$ , thereby completing the induction process. By Lemma E.1, we know that there exists time  $T^{\prime}$  such that
$$
\left. \left| \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} \right| \geq c _ {1} (\beta + 1) R _ {\min } ^ {2} / p n, \right.
$$
for any time  $t \geq T'$ . Taking  $T_2 = T'$  and  $c = 2 / c_1$ , we have
$$
| \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(t)} | \leq \beta + 1 + R _ {\min } ^ {- 2} p n | \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} | / c _ {1} \leq 2 R _ {\min } ^ {- 2} p n | \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} | / c _ {1} = c R _ {\min } ^ {- 2} p n | \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} |,
$$
Given Lemma E.2, the following corollary can be directly obtained.
Corollary E.3. There exists time  $T_{3}$  and constant  $c$  such that for any time  $t \geq T_{3}$
$$
\left| \underline {{\rho}} _ {y _ {i}, r, i ^ {\prime}} ^ {(t)} \right| \leq c R _ {\min } ^ {- 2} p n \left| \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} \right|, \forall r \in [ m ], r ^ {\prime} \in S _ {i} ^ {(0)}, i, i ^ {\prime} \in [ n ] \text {w i t h} - y _ {i} = y _ {i ^ {\prime}}.
$$
F Stable Rank of ReLU Network
In this section, we consider the properties of stable rank of the weight matrix  $\mathbf{W}^{(t)}$  found by gradient descent at time  $t$ , defined as  $\| \mathbf{W}^{(t)} \|_F^2 / \| \mathbf{W}^{(t)} \|_2^2$ . Given Lemma E.1, we have following coefficient update rule for any  $t \geq 0$ ,  $i \in [n]$  and  $r \in S_i^{(0)}$ :
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(t + 1)} = \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + \frac {\eta}{n m} \cdot \left| \ell_ {i} ^ {\prime (t)} \right| \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \tag {F.1}
$$
Now we are ready to prove the first bullet of Theorem 4.3.
Lemma F.1. For two-layer ReLU neural network defined in (E.1), under the same condition as Theorem 4.3, the stable rank of  $\mathbf{W}_j^{(t)}$  satisfies the following property:
$$
\limsup_{t\to \infty}\frac{\| \mathbf{W}_{j}^{(t)}\|_{F}^{2}}{\| \mathbf{W}_{j}^{(t)}\|_{2}^{2}}\leq C,
$$
where  $C = \Theta (1)$  is a constant.
Proof of Lemma F.1. By decomposition (5.1), we have
$$
\mathbf {w} _ {j, r} ^ {(t)} - \mathbf {w} _ {j, r} ^ {(0)} = \sum_ {i = 1} ^ {n} \rho_ {j, r, i} ^ {(t)} \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {- 2} \cdot \mathbf {x} _ {i} = \left[ \rho_ {j, r, 1} ^ {(t)} \| \mathbf {x} _ {1} \| _ {2} ^ {- 2}, \dots , \rho_ {j, r, n} ^ {(t)} \| \mathbf {x} _ {n} \| _ {2} ^ {- 2} \right] \cdot \left[ \begin{array}{c} \mathbf {x} _ {1} \\ \vdots \\ \mathbf {x} _ {n} \end{array} \right],
$$