SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} \leq \log \left(1 + \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {2 \beta}}{n m} \exp \left(\frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {2 \beta}}{n m}\right) \cdot t\right) \leq \log \left(1 + \frac {2 \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {- \beta}}{n m} \cdot t\right),
$$
$$
\frac {1}{m} \sum_ {r \in S _ {i} ^ {(0)}} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} \geq \log \left(1 + \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {- \beta}}{2 n m} \cdot t\right).
$$
$$
\lim_{t\to \infty}\frac{1}{m}\sum_{r\in S_{i}^{(0)}}\overline{\rho}_{y_{i},r,i}^{(t)} / \log t = 1.
$$
Since  $\overline{\rho}_{y_i,r,i}^{(t)} = \overline{\rho}_{y_i,r',i}^{(t)}$  for any  $r\neq r\in S_i^{(0)}$ , we have
$$
\lim _ {t \to \infty} \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} / \log t = m / | S _ {i} ^ {(0)} |,
$$
Lemma F.3. For two-layer ReLU neural network defined in (E.1), there exists mutually orthogonal data  $\mathbf{x}_1, \dots, \mathbf{x}_n$  such that stable rank of  $\mathbf{W}_j^{(t)}$  will converge to  $2 \pm o(1)$ .
Proof of Lemma F.3. By (5.1), we have
$$
\mathbf{w}_{j,r}^{(t)} = \mathbf{w}_{j,r}^{(0)} + \underbrace{\sum_{i = 1}^{n}\rho_{j,r,i}^{(t)}\cdot\|\mathbf{x}_{i}\|_{2}^{-2}\cdot\mathbf{x}_{i}}_{:= \mathbf{v}_{j,r}^{(t)}}.
$$
Given the definition of  $\mathbf{v}_{j,r}^{(t)}$ , we have the following representation of  $\mathbf{v}_{j,r}^{(t)}$  and  $\mathbf{V}_j^{(t)}$ .
$$
\mathbf {v} _ {j, r} ^ {(t)} = \left[ \rho_ {j, r, 1} ^ {(t)} \cdot \| \mathbf {x} _ {1} \| _ {2} ^ {- 2} \dots , \rho_ {j, r, n} ^ {(t)} \cdot \| \mathbf {x} _ {n} \| _ {2} ^ {- 2} \right] \cdot \left[ \begin{array}{c} \mathbf {x} _ {1} \\ \vdots \\ \mathbf {x} _ {n} \end{array} \right],
$$
$$
\mathbf {V} _ {j} ^ {(t)} = \left[ \begin{array}{c c c} \rho_ {j, r, 1} ^ {(t)} \cdot \| \mathbf {x} _ {1} \| _ {2} ^ {- 2} & \dots & \rho_ {j, r, n} ^ {(t)} \cdot \| \mathbf {x} _ {n} \| _ {2} ^ {- 2} \\ \vdots & \ddots & \vdots \\ \rho_ {j, m, 1} ^ {(t)} \cdot \| \mathbf {x} _ {1} \| _ {2} ^ {- 2} & \dots & \rho_ {j, m, n} ^ {(t)} \cdot \| \mathbf {x} _ {n} \| _ {2} ^ {- 2} \end{array} \right] \cdot \left[ \begin{array}{c} \mathbf {x} _ {1} \\ \vdots \\ \mathbf {x} _ {n} \end{array} \right].
$$
Assume  $n$  is an even number and  $\mathbf{x}_1, \dots, \mathbf{x}_{n/2}$  are with label  $+1$  while  $\mathbf{x}_{(n/2)+1}, \dots, \mathbf{x}_n$  are with label  $-1$ . And we take  $\mathbf{x}_1, \dots, \mathbf{x}_n$  as  $\mathbf{e}_1, \dots, \mathbf{e}_n$ . Given Lemma F.2,  $\mathbf{W}_j^{(t)} = \mathbf{W}_j^{(0)} + \mathbf{V}_j^{(t)}$  and such selection of training data, we have
$$
\lim _ {t \to \infty} \frac {\mathbf {W} _ {+ 1} ^ {(t)}}{\log t} = [ \mathbf {A} _ {m \times (n / 2)}, \mathbf {0} _ {m \times (n / 2)} ] \cdot [ \mathbf {I} _ {n}, \mathbf {0} _ {n \times (d - n)} ] = [ \mathbf {A} _ {m \times (n / 2)}, \mathbf {0} _ {m \times (d - (n / 2))} ],
$$
$$
\lim _ {t \to \infty} \frac {\mathbf {W} _ {- 1} ^ {(t)}}{\log t} = [ \mathbf {0} _ {m \times (n / 2)}, \mathbf {B} _ {m \times (n / 2)} ] \cdot [ \mathbf {I} _ {n}, \mathbf {0} _ {n \times (d - n)} ] = [ \mathbf {0} _ {m \times (n / 2)}, \mathbf {B} _ {m \times (n / 2)}, \mathbf {0} _ {n \times (d - n)} ],
$$
$$
\lim _ {t \to \infty} \frac {\mathbf {W} ^ {(t)}}{\log t} = \left[ \begin{array}{c c c} \mathbf {A} _ {m \times (n / 2)} & \mathbf {0} _ {m \times (n / 2)} & \mathbf {0} _ {n \times (d - n)} \\ \mathbf {0} _ {m \times (n / 2)} & \mathbf {B} _ {m \times (n / 2)} & \mathbf {0} _ {n \times (d - n)} \end{array} \right].
$$
$$
\mathbf {A} _ {m \times (n / 2)} = \underbrace {\left[ \begin{array}{c c c} \mathbb {1} [ \langle \mathbf {w} _ {+ 1 , 1} ^ {(0)} , \mathbf {x} _ {1} \rangle \geq 0 ] & \cdots & \mathbb {1} [ \langle \mathbf {w} _ {+ 1 , 1} ^ {(0)} , \mathbf {x} _ {n / 2} \rangle \geq 0 ] \\ \vdots & \ddots & \vdots \\ \mathbb {1} [ \langle \mathbf {w} _ {+ 1 , m} ^ {(0)} , \mathbf {x} _ {1} \rangle \geq 0 ] & \cdots & \mathbb {1} [ \langle \mathbf {w} _ {+ 1 , m} ^ {(0)} , \mathbf {x} _ {n / 2} \rangle \geq 0 ] \end{array} \right]} _ {:= \mathbf {C} _ {m \times (n / 2)}} \cdot \mathrm {d i a g} \left[ \begin{array}{c} m / | S _ {1} ^ {(0)} | \\ \vdots \\ m / | S _ {n / 2} ^ {(0)} | \end{array} \right],
$$
$$
\mathbf {B} _ {m \times (n / 2)} = \underbrace {\left[ \begin{array}{c c c} \mathbb {1} [ \langle \mathbf {w} _ {- 1 , 1} ^ {(0)} , \mathbf {x} _ {(n / 2) + 1} \rangle \geq 0 ] & \cdots & \mathbb {1} [ \langle \mathbf {w} _ {- 1 , 1} ^ {(0)} , \mathbf {x} _ {n} \rangle \geq 0 ] \\ \vdots & \ddots & \vdots \\ \mathbb {1} [ \langle \mathbf {w} _ {- 1 , m} ^ {(0)} , \mathbf {x} _ {(n / 2) + 1} \rangle \geq 0 ] & \cdots & \mathbb {1} [ \langle \mathbf {w} _ {- 1 , m} ^ {(0)} , \mathbf {x} _ {n} \rangle \geq 0 ] \end{array} \right]} _ {:= \mathbf {D} _ {m \times (n / 2)}} \cdot \mathrm {d i a g} \left[ \begin{array}{c} m / | S _ {(n / 2) + 1} ^ {(0)} | \\ \vdots \\ m / | S _ {n} ^ {(0)} | \end{array} \right].
$$
Then, we can get the stable rank limits as follows:
$$
\lim  _ {t \rightarrow \infty} \frac {\| \mathbf {W} _ {+ 1} ^ {(t)} \| _ {F} ^ {2}}{\| \mathbf {W} _ {+ 1} ^ {(t)} \| _ {2} ^ {2}} = \frac {\| \mathbf {A} \| _ {F} ^ {2}}{\| \mathbf {A} \| _ {2} ^ {2}},
$$
$$
\lim  _ {t \rightarrow \infty} \frac {\| \mathbf {W} _ {- 1} ^ {(t)} \| _ {F} ^ {2}}{\| \mathbf {W} _ {- 1} ^ {(t)} \| _ {2} ^ {2}} = \frac {\| \mathbf {B} \| _ {F} ^ {2}}{\| \mathbf {B} \| _ {2} ^ {2}},
$$
$$
\lim  _ {t \rightarrow \infty} \frac {\| \mathbf {W} ^ {(t)} \| _ {F} ^ {2}}{\| \mathbf {W} ^ {(t)} \| _ {2} ^ {2}} = \frac {\| \mathbf {A} \| _ {F} ^ {2} + \| \mathbf {B} \| _ {F} ^ {2}}{(\max  \{\| \mathbf {A} \| _ {2} , \| \mathbf {B} \| _ {2} \}) ^ {2}}.
$$
Since  $\mathbf{x}_1 = \mathbf{e}_1, \dots, \mathbf{x}_n = \mathbf{e}_n$ , we can get
$$
\mathbb {1} \left[ \left\langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \right\rangle \geq 0 \right] = \mathbb {1} \left[ \left[ \mathbf {w} _ {j, r} ^ {(0)} \right] _ {i} \geq 0 \right].
$$
Therefore, the entries of matrix  $\mathbf{C}$  and matrix  $\mathbf{D}$  can be regarded as i.i.d. random variables taking 0 or 1 with equal probability. For  $\| \mathbf{A}\| _F$  and  $\| \mathbf{B}\| _F$ , we have
$$
\| \mathbf {A} \| _ {F} ^ {2} = \sum_ {r = 1} ^ {m} \sum_ {i = 1} ^ {n / 2} \mathbb {1} \left[ \left[ \mathbf {w} _ {+ 1, r} ^ {(0)} \right] _ {i} \geq 0 \right] \cdot \left(m / \left| S _ {i} ^ {(0)} \right|\right) ^ {2},
$$
$$
\| \mathbf {B} \| _ {F} ^ {2} = \sum_ {r = 1} ^ {m} \sum_ {i = (n / 2) + 1} ^ {n} \mathbb {1} \left[ \left[ \mathbf {w} _ {- 1, r} ^ {(0)} \right] _ {i} \geq 0 \right] \cdot \left(m / \left| S _ {i} ^ {(0)} \right|\right) ^ {2}.
$$
By Lemma B.2, we have with probability at least  $1 - \delta$  that  $0.4m \leq |S_i^{(0)}| \leq 0.6m$ . By Hoeffding's inequality, we have with probability at least  $1 - 2\delta$  that
$$
\left| \| \mathbf {A} \| _ {F} ^ {2} - \frac {m}{2} \sum_ {i = 1} ^ {n / 2} (m / | S _ {i} ^ {(0)} |) ^ {2} \right| \leq \sqrt {\frac {m \log (2 / \delta)}{2} \sum_ {i = 1} ^ {n / 2} (m / | S _ {i} ^ {(0)} |) ^ {4}} \leq \sqrt {\frac {6 2 5 m n \log (2 / \delta)}{3 2}},
$$
$$
\left| \| \mathbf {B} \| _ {F} ^ {2} - \frac {m}{2} \sum_ {i = (n / 2) + 1} ^ {n} (m / | S _ {i} ^ {(0)} |) ^ {2} \right| \leq \sqrt {\frac {m \log (2 / \delta)}{2} \sum_ {i = (n / 2) + 1} ^ {n} (m / | S _ {i} ^ {(0)} |) ^ {4}} \leq \sqrt {\frac {6 2 5 m n \log (2 / \delta)}{3 2}}.
$$
Next, we estimate  $\| \mathbf{A}\| _2$  and  $\| \mathbf{B}\| _2$ . Let  $\mathbf{A} = \widetilde{\mathbf{A}} +\mathbb{E}[\mathbf{A}]$  and  $\mathbf{B} = \widetilde{\mathbf{B}} +\mathbb{E}[\mathbf{B}]$ . Assume  $\mathbf{G}$  be the  $m\times (n / 2)$  matrix with all entries equal to  $1 / 2$ . Then,
$$
\mathbb {E} [ \mathbf {A} ] = \mathbf {G} \cdot \operatorname {d i a g} \underbrace {\left[ \begin{array}{c} m / | S _ {1} ^ {(0)} | \\ \vdots \\ m / | S _ {n / 2} ^ {(0)} | \end{array} \right]} _ {:= \mathbf {a}}, \mathbb {E} [ \mathbf {B} ] = \mathbf {G} \cdot \operatorname {d i a g} \underbrace {\left[ \begin{array}{c} m / | S _ {(n / 2) + 1} ^ {(0)} | \\ \vdots \\ m / | S _ {n} ^ {(0)} | \end{array} \right]} _ {:= \mathbf {b}}.
$$
And the entries of matrix  $\widetilde{\mathbf{A}}$  and matrix  $\widetilde{\mathbf{B}}$  are independent, mean zero, sub-gaussian random variables. By Lemma H.3, we have with probability at least  $1 - \delta$  that
$$
\| \widetilde {\mathbf {A}} \| _ {2} \leq \frac {C}{2} (\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}),
$$
$$
\| \widetilde {\mathbf {B}} \| _ {2} \leq \frac {C}{2} \left(\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}\right),
$$
where  $C$  is a constant. Let  $\mathbf{1}_k$  denote the row vector with  $k$  entries equal to 1. Then for  $\mathbb{E}[\mathbf{A}]$  and  $\mathbb{E}[\mathbf{B}]$ , we have
$$
\begin{array}{l} \| \mathbb{E}[\mathbf{A}]\|_{2} = \max_{\mathbf{x}\in S^{\frac{n}{2} -1}}\| \mathbf{G}\mathrm{diag}(\mathbf{a})\mathbf{x}\|_{2} \\ = \max  _ {\mathbf {x} \in S ^ {\frac {n}{2} - 1}} \frac {1}{2} \| \mathbf {1} _ {m} ^ {\top} \mathbf {1} _ {\frac {n}{2}} \operatorname {d i a g} (\mathbf {a}) \mathbf {x} \| _ {2} \\ = \max  _ {\mathbf {x} \in S ^ {\frac {n}{2} - 1}} \frac {\sqrt {m}}{2} | \mathbf {1} _ {\frac {n}{2}} \operatorname {d i a g} (\mathbf {a}) \mathbf {x} | \\ = \max  _ {\mathbf {x} \in S ^ {\frac {n}{2} - 1}} \frac {\sqrt {m}}{2} | \mathbf {a} ^ {\top} \mathbf {x} | \\ = \frac {\sqrt {m} \| \mathbf {a} \| _ {2}}{2}, \\ \end{array}
$$
$$
\| \mathbb {E} [ \mathbf {B} ] \| _ {2} = \frac {\sqrt {m} \| \boldsymbol {b} \| _ {2}}{2}.
$$
By triangle inequality, we have
$$
\| \mathbf {A} \| _ {2} \geq \| \mathbf {C} \| _ {2} - \| \widetilde {\mathbf {A}} \| _ {2} \geq (\sqrt {m} \| \mathbf {a} \| _ {2}) / 2 - \frac {C}{2} \left(\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}\right),
$$
$$
\| \mathbf {A} \| _ {2} \leq \| \mathbf {C} \| _ {2} + \| \widetilde {\mathbf {A}} \| _ {2} \leq (\sqrt {m} \| \mathbf {a} \| _ {2}) / 2 + \frac {C}{2} \left(\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}\right),
$$
$$
\| \mathbf {B} \| _ {2} \geq \| \mathbf {C} \| _ {2} - \| \widetilde {\mathbf {B}} \| _ {2} \geq (\sqrt {m} \| \boldsymbol {b} \| _ {2}) / 2 - \frac {C}{2} \left(\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}\right),
$$
$$
\| \mathbf {B} \| _ {2} \leq \| \mathbf {C} \| _ {2} + \| \widetilde {\mathbf {B}} \| _ {2} \leq (\sqrt {m} \| \boldsymbol {b} \| _ {2}) / 2 + \frac {C}{2} \big (\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)} \big).
$$
Notice that  $\| \mathbf{a}\| _2^2 = \Theta (n)$  and  $\| \pmb {b}\| _2^2 = \Theta (n)$ , then with probability at least  $1 - 2\delta$ , we have
$$
\frac {\| \mathbf {A} \| _ {F} ^ {2}}{\| \mathbf {A} \| _ {2} ^ {2}} \leq \frac {m \| \mathbf {a} \| _ {2} ^ {2} / 2 + \sqrt {6 2 5 m n \log (2 / \delta) / 3 2}}{\left(\sqrt {m} \| \mathbf {a} \| _ {2} / 2 - \frac {C}{2} \left(\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}\right)\right) ^ {2}} = 2 + o (1),
$$
$$
\frac {\| \mathbf {A} \| _ {F} ^ {2}}{\| \mathbf {A} \| _ {2} ^ {2}} \geq \frac {m \| \mathbf {a} \| _ {2} ^ {2} / 2 - \sqrt {6 2 5 m n \log (2 / \delta) / 3 2}}{\left(\sqrt {m} \| \mathbf {a} \| _ {2} / 2 + \frac {C}{2} \left(\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}\right)\right) ^ {2}} = 2 - o (1),
$$
$$
\frac {\| \mathbf {B} \| _ {F} ^ {2}}{\| \mathbf {B} \| _ {2} ^ {2}} \geq \frac {m \| \boldsymbol {b} \| _ {2} ^ {2} / 2 + \sqrt {6 2 5 m n \log (2 / \delta) / 3 2}}{\left(\sqrt {m} \| \boldsymbol {b} \| _ {2} / 2 - \frac {C}{2} \left(\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}\right)\right) ^ {2}} = 2 + o (1),
$$
$$
\frac {\| \mathbf {B} \| _ {F} ^ {2}}{\| \mathbf {B} \| _ {2} ^ {2}} \geq \frac {m \| \boldsymbol {b} \| _ {2} ^ {2} / 2 - \sqrt {6 2 5 m n \log (2 / \delta) / 3 2}}{\left(\sqrt {m} \| \boldsymbol {b} \| _ {2} / 2 + \frac {C}{2} \left(\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}\right)\right) ^ {2}} = 2 - o (1).
$$
$$
\lim  _ {t \to \infty} \frac {\| \mathbf {W} _ {+ 1} ^ {(t)} \| _ {F} ^ {2}}{\| \mathbf {W} _ {+ 1} ^ {(t)} \| _ {2} ^ {2}} = \frac {\| \mathbf {A} \| _ {F} ^ {2}}{\| \mathbf {A} \| _ {2} ^ {2}} = 2 \pm o (1),
$$
$$
\lim _ {t \to \infty} \frac {\| \mathbf {W} _ {- 1} ^ {(t)} \| _ {F} ^ {2}}{\| \mathbf {W} _ {- 1} ^ {(t)} \| _ {2} ^ {2}} = \frac {\| \mathbf {B} \| _ {F} ^ {2}}{\| \mathbf {B} \| _ {2} ^ {2}} = 2 \pm o (1).
$$
For  $\mathbf{W}^{(t)}$ , we have the following lower bound
$$
\begin{array}{l} \frac {\| \mathbf {A} \| _ {F} ^ {2} + \| \mathbf {B} \| _ {F} ^ {2}}{(\max  \{\| \mathbf {A} \| _ {2} , \| \mathbf {B} \| _ {2} \}) ^ {2}} \geq \frac {m (\| \mathbf {a} \| _ {2} ^ {2} + \| \mathbf {b} \| _ {2} ^ {2}) / 2 - \sqrt {6 2 5 m n \log (2 / \delta) / 8}}{\left(\sqrt {m} \max  \{\| \mathbf {a} \| _ {2} , \| \mathbf {b} \| _ {2} \} / 2 + \frac {C}{2} \left(\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)}\right)\right) ^ {2}} \\ \geq (2 - o (1)) \cdot \frac {\| \mathbf {a} \| _ {2} ^ {2} + \| \boldsymbol {b} \| _ {2} ^ {2}}{(\max  \{\| \mathbf {a} \| _ {2} , \| \boldsymbol {b} \| _ {2} \}) ^ {2}} \\ \end{array}
$$
$$
\geq \frac {1 6}{9} - o (1),
$$
where the third inequality is by  $(5/3)\sqrt{n} \leq \| \mathbf{a} \|_2 \leq (5/2)\sqrt{n}$  and  $(5/3)\sqrt{n} \leq \| \mathbf{b} \|_2 \leq (5/2)\sqrt{n}$  due to  $0.4m \leq |S_i^{(0)}| \leq 0.6m$ . And
$$
\begin{array}{l} \frac {\| \mathbf {A} \| _ {F} ^ {2} + \| \mathbf {B} \| _ {F} ^ {2}}{(\max  \{\| \mathbf {A} \| _ {2} , \| \mathbf {B} \| _ {2} \}) ^ {2}} \leq \frac {m (\| \mathbf {a} \| _ {2} ^ {2} + \| \mathbf {b} \| _ {2} ^ {2}) / 2 + \sqrt {6 2 5 m n \log (2 / \delta) / 8}}{(\sqrt {m} \max  \{\| \mathbf {a} \| _ {2} , \| \mathbf {b} \| _ {2} \} / 2 - \frac {C}{2} (\sqrt {m} + \sqrt {n} + \sqrt {\log (2 / \delta)})) ^ {2}} \\ \leq (2 + o (1)) \cdot \frac {\| \mathbf {a} \| _ {2} ^ {2} + \| \boldsymbol {b} \| _ {2} ^ {2}}{(\max  \{\| \mathbf {a} \| _ {2} , \| \boldsymbol {b} \| _ {2} \}) ^ {2}} \\ \leq 9 + o (1), \\ \end{array}
$$
where the third inequality is by  $(5/3)\sqrt{n} \leq \| \mathbf{a}\|_2 \leq (5/2)\sqrt{n}$  and  $(5/3)\sqrt{n} \leq \| \mathbf{b}\|_2 \leq (5/2)\sqrt{n}$  due to  $0.4m \leq |S_i^{(0)}| \leq 0.6m$ . Therefore,
$$
\lim  _ {t \rightarrow \infty} \frac {\| \mathbf {W} \| _ {F} ^ {2}}{\| \mathbf {W} \| _ {2} ^ {2}} = \frac {\| \mathbf {A} \| _ {F} ^ {2} + \| \mathbf {B} \| _ {F} ^ {2}}{(\max  \{\| \mathbf {A} \| _ {2} , \| \mathbf {B} \| _ {2} \}) ^ {2}} \in [ 1 6 / 9 - o (1), 9 + o (1) ].
$$
G Margin Results and Loss Convergence
In this section, we prove the convergence rate of training loss as well as the increasing rate of margin for both two-layer ReLU and leaky ReLU networks defined as
$$
\begin{array}{l} f (\mathbf {W} ^ {(t)}, \mathbf {x}) = F _ {+ 1} (\mathbf {W} _ {+ 1} ^ {(t)}, \mathbf {x}) - F _ {- 1} (\mathbf {W} _ {- 1} ^ {(t)}, \mathbf {x}) \\ = \frac {1}{m} \sum_ {r = 1} ^ {m} \sigma \left(\left\langle \mathbf {w} _ {+ 1, r} ^ {(t)}, \mathbf {x} \right\rangle\right) - \frac {1}{m} \sum_ {r = 1} ^ {m} \sigma \left(\left\langle \mathbf {w} _ {- 1, r} ^ {(t)}, \mathbf {x} \right\rangle\right), \tag {G.1} \\ \sigma \in \{\text {R e L U}, \text {l e a k y} \text {R e L U} \}. \\ \end{array}
$$
We first prove the following auxiliary lemma.
Lemma G.1. For both two-layer leaky ReLU and ReLU neural networks defined in (G.1), the following properties hold for any  $t \geq 0$ :
-  $y_{i}f(\mathbf{W}^{(t)},\mathbf{x}_{i}) \geq -c$  for any  $i \in [n]$  where  $c$  is a positive constant.
-  $y_{i}f(\mathbf{W}^{(t)},\mathbf{x}_{i}) - y_{k}f(\mathbf{W}^{(t)},\mathbf{x}_{k}) \leq C_{1}$  for any  $i, k \in [n]$  where  $C_1$  is a constant.
-  $\ell_i^{\prime (t)} / \ell_k^{\prime (t)}\leq C_2$  for any  $i,k\in [n]$  where  $C_2$  is a constant.
-  $S_{i}^{(t)} \subseteq S_{i}^{(t + 1)}$  for any  $i \in [n]$ , where  $S_{i}^{(t)} \coloneqq \{r \in [m] : \langle \mathbf{w}_{y_{i}, r}^{(t)}, \mathbf{x}_{i} \rangle \geq 0\}$ .
Proof of Lemma G.1. We prove this lemma by induction. When  $t = 0$ , since
$$
\begin{array}{l} \left| y _ {i} f \left(\mathbf {W} ^ {(0)}, \mathbf {x} _ {i}\right) \right| = \left| \sum_ {j} j y _ {i} F _ {j} \left(\mathbf {W} _ {j} ^ {(0)}, \mathbf {x} _ {i}\right) \right| \\ = \left| \sum_ {j} j y _ {i} \cdot \frac {1}{m} \sum_ {r = 1} ^ {m} \sigma \left(\left\langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \right\rangle\right) \right| \\ \leq \sum_ {j} \frac {1}{m} \sum_ {r = 1} ^ {m} | \sigma (\langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle) | \\ \leq \sum_ {j} \frac {1}{m} \sum_ {r = 1} ^ {m} | \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle | \\ \leq 2 \beta , \\ \end{array}
$$
the first bullet holds as long as  $c \geq 2\beta$ . We also have
$$
y _ {i} f (\mathbf {W} ^ {(0)}, \mathbf {x} _ {i}) - y _ {k} f (\mathbf {W} ^ {(0)}, \mathbf {x} _ {k}) \leq | y _ {i} f (\mathbf {W} ^ {(t)}, \mathbf {x} _ {i}) | + | y _ {k} f (\mathbf {W} ^ {(0)}, \mathbf {x} _ {k}) | \leq 4 \beta ,
$$
which verifies the second bullet at time  $t = 0$  as long as  $C_1 \geq 4\beta$ . This leads to
$$
\begin{array}{l} \frac {\ell_ {i} ^ {\prime (0)}}{\ell_ {k} ^ {\prime (0)}} = \frac {1 + \exp (y _ {k} f (\mathbf {W} ^ {(0)} , \mathbf {x} _ {k}))}{1 + \exp (y _ {i} f (\mathbf {W} ^ {(0)} , \mathbf {x} _ {i}))} \\ \leq 2 \left(1 + \exp \left(y _ {k} f \left(\mathbf {W} ^ {(0)}, \mathbf {x} _ {k}\right) - y _ {i} f \left(\mathbf {W} ^ {(0)}, \mathbf {x} _ {i}\right)\right)\right) \\ \leq 2 \left(1 + \exp \left(C _ {1}\right)\right) \\ \leq C _ {2}, \\ \end{array}
$$
as long as  $C_2 \geq 2(1 + \exp(C_1))$ . For any  $r \in S_i^{(0)}$ , we have
$$
\begin{array}{l} \langle \mathbf {w} _ {y _ {i}, r} ^ {(1)}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \frac {\eta}{n m} \sum_ {i ^ {\prime} = 1} ^ {n} | \ell_ {i ^ {\prime}} ^ {\prime (0)} | \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle) \cdot \langle y _ {i ^ {\prime}} \mathbf {x} _ {i ^ {\prime}}, y _ {i} \mathbf {x} _ {i} \rangle \\ = \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \frac {\eta}{n m} \cdot | \ell_ {i} ^ {\prime (0)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} + \frac {\eta}{n m} \sum_ {i ^ {\prime} \neq i} | \ell_ {i ^ {\prime}} ^ {\prime (0)} | \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle) \cdot \langle y _ {i ^ {\prime}} \mathbf {x} _ {i ^ {\prime}}, y _ {i} \mathbf {x} _ {i} \rangle \\ \geq \langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \frac {\eta}{n m} \cdot | \ell_ {i} ^ {\prime (0)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} - \frac {\eta}{n m} \sum_ {i ^ {\prime} \neq i} | \ell_ {i ^ {\prime}} ^ {\prime (0)} | \cdot | \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle | \\ \geq \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(0)}, \mathbf {x} _ {i} \right\rangle \geq 0, \\ \end{array}
$$
where the second equality is by  $\langle \mathbf{w}_{y_i,r}^{(0)},\mathbf{x}_i\rangle \geq 0$ ; the first inequality is by triangle inequality; the second inequality is by  $|\ell_{i^{\prime}}^{\prime (0)}| / |\ell_i^{\prime (0)}|\leq C_2$  and the condition that  $R_{\mathrm{min}}^2\geq Cnp$ ,  $C$  is a sufficiently large constant. This verifies the fourth bullet at time  $t = 0$ .
Now suppose there exists time  $\widetilde{t} \geq 0$  such that these four hypotheses hold for any  $0 \leq t \leq \widetilde{t}$ . We aim to prove that these conditions also hold for  $t = \widetilde{t} + 1$ . We first prove that  $y_{i}f(\mathbf{W}^{(\widetilde{t} + 1)}, \mathbf{x}_{i}) \geq y_{i}f(\mathbf{W}^{(\widetilde{t})}, \mathbf{x}_{i})$ . We have
By the fourth induction hypothesis at time  $t = \widetilde{t}$ , we have  $S_{i}^{(\widetilde{t} +1)}\subseteq S_{i}^{(\widetilde{t})}$  and hence
$$
\frac {\sigma \left(\left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t} + 1)} , \mathbf {x} _ {i} \right\rangle\right) - \sigma \left(\left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t})}, \mathbf {x} _ {i} \right\rangle\right)}{\left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t} + 1)}, \mathbf {x} _ {i} \right\rangle - \left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t})}, \mathbf {x} _ {i} \right\rangle} = \frac {\left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t} + 1)} , \mathbf {x} _ {i} \right\rangle - \left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t})} , \mathbf {x} _ {i} \right\rangle}{\left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t} + 1)}, \mathbf {x} _ {i} \right\rangle - \left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t})}, \mathbf {x} _ {i} \right\rangle} = 1, \tag {G.2}
$$
for  $j = y_{i}$  and  $r \in S_{i}^{(t)}$ . For  $\sigma \in \{\mathrm{ReLU}, \text{leakyReLU}\}$ ,  $\sigma$  is non-decreasing and 1-Lipschitz continuous, which gives
$$
0 \leq \frac {\sigma \left(\left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t} + 1)} , \mathbf {x} _ {i} \right\rangle\right) - \sigma \left(\left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t})}, \mathbf {x} _ {i} \right\rangle\right)}{\left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t} + 1)}, \mathbf {x} _ {i} \right\rangle - \left\langle \mathbf {w} _ {j , r} ^ {(\widetilde {t})}, \mathbf {x} _ {i} \right\rangle} \leq 1. \tag {G.3}
$$
$$
\begin{array}{l} y _ {i} f (\mathbf {W} ^ {(\widetilde {t} + 1)}, \mathbf {x} _ {i}) - y _ {i} f (\mathbf {W} ^ {(\widetilde {t})}, \mathbf {x} _ {i}) \geq \frac {\eta}{n m ^ {2}} \sum_ {r \in S _ {i} ^ {(\widetilde {t})}} | \ell_ {i} ^ {\prime (\widetilde {t})} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} - \frac {\eta}{n m ^ {2}} \sum_ {j} \sum_ {r = 1} ^ {m} \sum_ {i ^ {\prime} \neq i} | \ell_ {i ^ {\prime}} ^ {\prime (\widetilde {t})} | \cdot | \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle | \\ \geq \frac {\eta}{2 n m ^ {2}} \sum_ {r \in S _ {i} ^ {(\tilde {t})}} | \ell_ {i} ^ {\prime (\tilde {t})} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \\ = \frac {\eta | S _ {i} ^ {(\widetilde {t})} |}{2 n m ^ {2}} \cdot | \ell_ {i} ^ {\prime (\widetilde {t})} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \\ \geq \frac {\eta}{5 n m} \cdot | \ell_ {i} ^ {(\widetilde {t})} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \\ \end{array}
$$
where the first inequality is by (G.2), (G.3),  $\sigma' \in [0,1]$  and triangle inequality; the second inequality is by  $|\ell_{i'}^{\prime(\widetilde{t})}| / |\ell_{i}'(\widetilde{t})| \leq C_2$ ,  $|S_i^{(\widetilde{t})}| \geq |S_i^{(0)}| \geq 0.4m$  and the condition that  $R_{\min}^2 \geq Cnp$ ,  $C$  is a sufficiently large constant. And
$$
\begin{array}{l} y _ {i} f (\mathbf {W} ^ {(\widetilde {t} + 1)}, \mathbf {x} _ {i}) - y _ {i} f (\mathbf {W} ^ {(\widetilde {t})}, \mathbf {x} _ {i}) \\ \leq \sum_ {j} \sum_ {r = 1} ^ {m} \frac {\eta}{n m ^ {2}} \sum_ {r \in S _ {i} ^ {(t)}} | \ell_ {i} ^ {\prime (\widetilde {t})} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} + \frac {\eta}{n m ^ {2}} \sum_ {j} \sum_ {r = 1} ^ {m} \sum_ {i ^ {\prime} \neq i} | \ell_ {i ^ {\prime}} ^ {\prime (\widetilde {t})} | \cdot | \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle | \\ \leq \frac {3 \eta}{2 n m ^ {2}} \sum_ {j} \sum_ {r = 1} ^ {m} | \ell_ {i} ^ {\prime (\tilde {t})} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2} \\ = \frac {3 \eta}{n m} \cdot | \ell_ {i} ^ {\prime (\widetilde {t})} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \\ \end{array}
$$
where the first inequality is by (G.3),  $\sigma' \in [0,1]$  and triangle inequality; the second inequality is by  $|\ell_{i'}^{\prime(\widetilde{t})}| / |\ell_i^{\prime(\widetilde{t})}| \leq C_2$ ,  $|S_i^{(\widetilde{t})}| \geq |S_i^{(0)}| \geq 0.4m$  and the condition that  $R_{\min}^2 \geq Cnp$ ,  $C$  is a sufficiently large constant. Now, we obtain
$$
y _ {i} f \left(\mathbf {W} ^ {\left(\widetilde {t} + 1\right)}, \mathbf {x} _ {i}\right) \geq y _ {i} f \left(\mathbf {W} ^ {\left(\widetilde {t}\right)}, \mathbf {x} _ {i}\right) + \frac {\eta}{5 n m} \cdot \left| \ell_ {i} ^ {\prime \left(\widetilde {t}\right)} \right| \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \tag {G.4}
$$