SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\left| \ell_ {i} ^ {\prime (t)} \right| \leq \exp \left\{\frac {\gamma \left(1 - \gamma^ {- 3} c _ {3} ^ {- 1} R _ {\operatorname* {m i n}} ^ {- 2} p n\right)}{m} \sum_ {r = 1} ^ {m} \rho_ {- y _ {i}, r, i} ^ {(t)} + 2 \beta \right\}. \tag {C.23}
$$
By (C.21) and  $\gamma \rho_{y_i,r,i}^{(t)}\leq |\rho_{-y_i,r,i}^{(t)}|\leq \gamma^{-4}\rho_{y_i,r,i}^{(t)}$ , we can get for  $0\leq t\leq \widetilde{t} -1$  that
$$
\left| \ell_ {i} ^ {\prime (t)} \right| \geq \frac {1}{2} \exp \left\{- \frac {2 \left(1 + \gamma^ {- 2} c _ {3} ^ {- 1} R _ {\operatorname* {m i n}} ^ {- 2} p n\right)}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)} - (\gamma + 1) \beta \right\}, \tag {C.24}
$$
$$
\begin{array}{l} | \ell_ {i} ^ {\prime (t)} | \geq \frac {1}{2} \exp \left\{\frac {(\gamma^ {- 1} + \gamma) (1 + \gamma^ {- 2} c _ {3} ^ {- 1} R _ {\mathrm {m i n}} ^ {- 2} p n)}{m} \sum_ {r = 1} ^ {m} \rho_ {- y _ {i}, r, i} ^ {(t)} - (\gamma + 1) \beta \right\} \\ \geq \frac {1}{2} \exp \left\{\frac {2 \left(1 + \gamma^ {- 2} c _ {3} ^ {- 1} R _ {\operatorname* {m i n}} ^ {- 2} p n\right)}{\gamma m} \sum_ {r = 1} ^ {m} \rho_ {- y _ {i}, r, i} ^ {(t)} - (\gamma + 1) \beta \right\}. \tag {C.25} \\ \end{array}
$$
By (5.4), (5.5) and  $\sigma^{\prime}\in [\gamma ,1]$ , we have for  $0\leq t\leq \widetilde{t} -1$  that
$$
\begin{array}{l} \rho_ {y _ {i}, r, i} ^ {(t + 1)} \leq \rho_ {y _ {i}, r, i} ^ {(t)} + \frac {\eta}{n m} \cdot | \ell_ {i} ^ {\prime (t)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \\ \rho_ {y _ {i}, r, i} ^ {(t + 1)} \geq \rho_ {y _ {i}, r, i} ^ {(t)} + \frac {\gamma \eta}{n m} \cdot \left| \ell_ {i} ^ {\prime (t)} \right| \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, (C.26) \\ \left| \rho_ {- y _ {i}, r, i} ^ {(t + 1)} \right| \leq \left| \rho_ {- y _ {i}, r, i} ^ {(t)} \right| + \frac {\eta}{n m} \cdot \left| \ell_ {i} ^ {\prime (t)} \right| \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, (C.20) \\ | \rho_ {- y _ {i}, r, i} ^ {(t + 1)} | \geq | \rho_ {- y _ {i}, r, i} ^ {(t)} | + \frac {\gamma \eta}{n m} \cdot | \ell_ {i} ^ {\prime (t)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}. \\ \end{array}
$$
By plugging (C.22), (C.24), (C.23) and (C.25) into (C.26), we have for  $0 \leq t \leq \widetilde{t} - 1$  that
$$
\sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t + 1)} \leq \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)} + \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {2 \beta}}{n} \cdot \exp \left\{- \frac {1 - \gamma^ {- 2} c _ {3} ^ {- 1} R _ {\operatorname* {m i n}} ^ {- 2} p n}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)} \right\}, \tag {C.27}
$$
$$
\sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t + 1)} \right| \leq \sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t)} \right| + \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {2 \beta}}{n} \cdot \exp \left\{- \frac {\gamma (1 - \gamma^ {- 3} c _ {3} ^ {- 1} R _ {\operatorname* {m i n}} ^ {- 2} p n)}{m} \sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t)} \right| \right\}, \tag {C.28}
$$
$$
\sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t + 1)} \geq \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)} + \frac {\gamma \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {- (\gamma + 1) \beta}}{2 n} \cdot \exp \left\{- \frac {2 (1 + \gamma^ {- 2} c _ {3} ^ {- 1} R _ {\operatorname* {m i n}} ^ {- 2} p n)}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)} \right\}, \tag {C.29}
$$
$$
\sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t + 1)} \right| \geq \sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t)} \right| + \frac {\gamma \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} e ^ {- (\gamma + 1) \beta}}{2 n} \cdot \exp \left\{- \frac {2 (1 + \gamma^ {- 2} c _ {3} ^ {- 1} R _ {\operatorname* {m i n}} ^ {- 2} p n)}{\gamma m} \sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t)} \right| \right\}. \tag {C.30}
$$
By applying Lemma H.1 to (C.27) and taking
$$
x _ {t} = \frac {1 - \gamma^ {- 2} c _ {3} ^ {- 1} R _ {\mathrm {m i n}} ^ {- 2} p n}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)},
$$
we can get for  $0 \leq t \leq \widetilde{t}$  that
$$
\begin{array}{l} \frac {1}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)} \leq c _ {4} ^ {- 1} \log \left(1 + \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} c _ {4} e ^ {2 \beta}}{n m} \exp \left\{\frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} c _ {4} e ^ {2 \beta}}{n m} \right\} \cdot t\right) \tag {C.31} \\ \leq c _ {4} ^ {- 1} \log \left(1 + \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} c _ {4} e ^ {2 \beta}}{n m} \cdot t\right), \\ \end{array}
$$
where  $c_4 \coloneqq 1 - \gamma^{-2}c_3^{-1}R_{\mathrm{min}}^{-2}pn$  and the last inequality is by  $\eta \leq (CR_{\mathrm{max}}^2 /nm)^{-1}$  and  $C$  is a sufficiently large constant.
By applying Lemma H.1 to (C.28) and taking
$$
x _ {t} = \frac {\gamma (1 - \gamma^ {- 3} c _ {3} ^ {- 1} R _ {\operatorname* {m i n}} ^ {- 2} p n)}{m} \sum_ {r = 1} ^ {m} | \rho_ {- y _ {i}, r, i} ^ {(t)} |,
$$
$$
\begin{array}{l} \frac {1}{m} \sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t)} \right| \leq c _ {5} ^ {- 1} \gamma^ {- 1} \log \left(1 + \frac {\gamma \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} c _ {5} e ^ {2 \beta}}{n m} \exp \left\{\frac {\gamma \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} c _ {5} e ^ {2 \beta}}{n m} \right\} \cdot t\right) \tag {C.32} \\ \leq c _ {5} ^ {- 1} \gamma^ {- 1} \log \left(1 + \frac {\gamma \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} c _ {5} e ^ {2 \beta}}{n m} \cdot t\right), \\ \end{array}
$$
where  $c_{5} \coloneqq 1 - \gamma^{-3}c_{3}^{-1}R_{\mathrm{min}}^{-2}pn$  and the last inequality is by  $\eta \leq (CR_{\mathrm{max}}^2 /nm)^{-1}$  and  $C$  is a sufficiently large constant.
By applying Lemma H.2 to (C.29) and taking
$$
x _ {t} = \frac {2 (1 + \gamma^ {- 2} c _ {3} ^ {- 1} R _ {\mathrm {m i n}} ^ {- 2} p n)}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)},
$$
$$
\frac {1}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)} \geq (2 c _ {6}) ^ {- 1} \log \left(1 + \frac {\gamma \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} c _ {6} e ^ {- (\gamma + 1) \beta}}{n m} \cdot t\right), \tag {C.33}
$$
where  $c_6 \coloneqq 1 + \gamma^{-2}c_3^{-1}R_{\min}^{-2}pn.$
By applying Lemma H.2 to (C.30) and taking
$$
x _ {t} = \frac {2 (1 + \gamma^ {- 4} c _ {3} ^ {- 1} R _ {\mathrm {m i n}} ^ {- 2} p n)}{\gamma m} \sum_ {r = 1} ^ {m} | \rho_ {- y _ {i}, r, i} ^ {(t)} |,
$$
$$
\frac {1}{m} \sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t)} \right| \geq (2 c _ {6}) ^ {- 1} \gamma \log \left(1 + \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2} c _ {6} e ^ {- (\gamma + 1) \beta}}{n m} \cdot t\right), \tag {C.34}
$$
where  $c_{6} \coloneqq 1 + \gamma^{-2} c_{3}^{-1} R_{\min}^{-2} p n$ .
In order to apply Lemma H.4 (requiring  $b > a$ ), we loosen the bounds in (C.31), (C.32), (C.33) and (C.34) as follows:
$$
\frac {1}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)} \leq c _ {4} ^ {- 1} \gamma^ {- 1} \log \left(1 + \frac {\gamma \eta R _ {\max } ^ {2} c _ {5} e ^ {2 \beta}}{n m} \cdot t\right), \forall 0 \leq t \leq \widetilde {t}, \tag {C.35}
$$
$$
\frac {1}{m} \sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t)} \right| \leq c _ {4} ^ {- 1} \gamma^ {- 1} \log \left(1 + \frac {\gamma \eta R _ {\max } ^ {2} c _ {5} e ^ {2 \beta}}{n m} \cdot t\right), \forall 0 \leq t \leq \widetilde {t}, \tag {C.36}
$$
$$
\frac {1}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i}, r, i} ^ {(t)} \geq (2 c _ {6}) ^ {- 1} \gamma \log \left(1 + \frac {\eta R _ {\min } ^ {2} c _ {6} e ^ {- (\gamma + 1) \beta}}{n m} \cdot t\right), \forall 0 \leq t \leq \widetilde {t}, \tag {C.37}
$$
$$
\frac {1}{m} \sum_ {r = 1} ^ {m} \left| \rho_ {- y _ {i}, r, i} ^ {(t)} \right| \geq (2 c _ {6}) ^ {- 1} \gamma \log \left(1 + \frac {\eta R _ {\min } ^ {2} c _ {6} e ^ {- (\gamma + 1) \beta}}{n m} \cdot t\right), \forall 0 \leq t \leq \widetilde {t}, \tag {C.38}
$$
where (C.35) is by Bernoulli's inequality that  $1 + \gamma^{-1}x\leq (1 + x)^{\gamma^{-1}}$  for every real number  $0\leq r\leq 1$  and  $x\geq -1$ ; (C.37) is by Bernoulli's inequality that  $1 + \gamma x\geq (1 + x)^{\gamma}$  for every real number  $0\leq r\leq 1$  and  $x\geq -1$ . If  $R_{\mathrm{min}}^2 c_6e^{-(\gamma +1)\beta}\geq \gamma R_{\mathrm{max}}^2 c_5e^{2\beta}$ , we have
$$
\frac {1}{m} \sum_ {r = 1} ^ {m} \left| \rho_ {j, r, i} ^ {(t)} \right| \geq \frac {\gamma^ {2} (2 c _ {6}) ^ {- 1} c _ {4}}{m} \sum_ {r = 1} ^ {m} \left| \rho_ {j ^ {\prime}, r, i ^ {\prime}} ^ {(t)} \right|. \tag {C.39}
$$
If  $R_{\mathrm{min}}^2 c_6e^{-(\gamma +1)\beta} < \gamma R_{\mathrm{max}}^2 c_5e^{2\beta}$ , by Lemma H.4, we have
$$
\begin{array}{l} \frac {\operatorname* {m i n} \left\{\frac {1}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i} , r , i} ^ {(t)} , \frac {1}{m} \sum_ {r = 1} ^ {m} | \rho_ {- y _ {i} , r , i} ^ {(t)} | \right\}}{\operatorname* {m a x} \left\{\frac {1}{m} \sum_ {r = 1} ^ {m} \rho_ {y _ {i ^ {\prime}} , r , i ^ {\prime}} ^ {(t)} , \frac {1}{m} \sum_ {r = 1} ^ {m} | \rho_ {- y _ {i ^ {\prime}} , r , i ^ {\prime}} ^ {(t)} | \right\}} \\ \geq \gamma^ {2} (2 c _ {6}) ^ {- 1} c _ {4} \cdot \frac {\log \left(1 + \frac {\eta R _ {\mathrm {m i n}} ^ {2} c _ {6} e ^ {- (\gamma + 1) \beta}}{n m} \cdot t\right)}{\log \left(1 + \frac {\gamma \eta R _ {\mathrm {m a x}} ^ {2} c _ {5} e ^ {2 \beta}}{n m} \cdot t\right)} \\ \geq \gamma^ {2} (2 c _ {6}) ^ {- 1} c _ {4} \cdot \frac {\log \left(1 + \frac {\eta R _ {\mathrm {m i n}} ^ {2} c _ {6} e ^ {- (\gamma + 1) \beta}}{n m} \cdot T _ {1}\right)}{\log \left(1 + \frac {\gamma \eta R _ {\mathrm {m a x}} ^ {2} c _ {5} e ^ {2 \beta}}{n m} \cdot T _ {1}\right)} \\ \geq \gamma^ {2} (2 c _ {6}) ^ {- 1} c _ {4} \cdot \frac {\log (1 + R ^ {- 2} c _ {6} e ^ {- (\gamma + 1) \beta} C ^ {\prime})}{\log (1 + \gamma c _ {5} e ^ {2 \beta} C ^ {\prime})}. \\ \end{array}
$$
Therefore, we can get for  $0 \leq t \leq \widetilde{t}$  that
$$
\sum_ {r = 1} ^ {m} \left| \rho_ {j, r, i} ^ {(t)} \right| \geq \gamma^ {2} c _ {3} \sum_ {r = 1} ^ {m} \left| \rho_ {j ^ {\prime}, r, i ^ {\prime}} ^ {(t)} \right|, \forall j, j ^ {\prime} \in \{\pm 1 \}, \forall i, i ^ {\prime} \in [ n ], \tag {C.40}
$$
$$
c _ {3} \leq (2 c _ {6}) ^ {- 1} c _ {4} \cdot \min  \left\{1, \frac {\log (1 + R ^ {- 2} c _ {6} e ^ {- (\gamma + 1) \beta} C ^ {\prime})}{\log (1 + \gamma c _ {5} e ^ {2 \beta} C ^ {\prime})} \right\}.
$$
This condition holds under the following conditions:
$$
\gamma^ {- 3} c _ {3} ^ {- 1} R _ {\min } ^ {- 2} p n \leq \frac {1}{2} \quad \Longrightarrow \quad c _ {4}, c _ {5} \geq \frac {1}{2}, c _ {6} \leq \frac {3}{2},
$$
$$
c _ {3} = \frac {1}{6} \min \left\{1, \frac {\log (1 + R ^ {- 2} e ^ {- (\gamma + 1) \beta} C ^ {\prime})}{\log (1 + \gamma e ^ {2 \beta} C ^ {\prime})} \right\}.
$$
This implies that induction hypothesis (C.15) holds for  $t = \widetilde{t}$ .
Lemma C.4 (Implication of Lemma C.1). Under the same condition as Theorem 4.1, if (C.8) hold for time  $t$ , then we have that
$$
| \rho_ {j, r, i} ^ {(t)} | \geq c _ {1} \gamma^ {4} | \rho_ {j ^ {\prime}, r ^ {\prime}, i ^ {\prime}} ^ {(t)} |,
$$
where  $c_{1}$  is the same constant as defined in Lemma C.1.
Proof of Lemma C.4. By  $\sigma' \in [\gamma, 1]$ , (5.4) and (5.5), we have
$$
| \rho_ {j, r, i} ^ {(t + 1)} | \geq | \rho_ {j, r, i} ^ {(t)} | + \frac {\gamma \eta}{n m} \cdot | \ell_ {i} ^ {\prime (t)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \forall j \in \{\pm 1 \}, \forall r \in [ m ], \forall i \in [ n ],
$$
$$
| \rho_ {j, r, i} ^ {(t + 1)} | \leq | \rho_ {j, r, i} ^ {(t)} | + \frac {\eta}{n m} \cdot | \ell_ {i} ^ {\prime (t)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \forall j \in \{\pm 1 \}, \forall r \in [ m ], \forall i \in [ n ].
$$
$$
| \rho_ {j, r, i} ^ {(t)} | \geq \frac {\gamma \eta \| \mathbf {x} _ {i} \| _ {2} ^ {2}}{n m} \cdot \sum_ {s = 1} ^ {t - 1} | \ell_ {i} ^ {\prime (t)} |, \forall j \in \{\pm 1 \}, \forall r \in [ m ], \forall i \in [ n ],
$$
$$
| \rho_ {j, r, i} ^ {(t)} | \leq \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2}}{n m} \cdot \sum_ {s = 1} ^ {t - 1} | \ell_ {i} ^ {\prime (t)} |, \forall j \in \{\pm 1 \}, \forall r \in [ m ], \forall i \in [ n ].
$$
Therefore,  $|\rho_{j,r,i}^{(t)}| \geq \gamma |\rho_{j',r',i}^{(t)}|$  for any  $j,j' \in \{\pm 1\}, r', r \in [m]$  and  $i \in [n]$ , and hence
$$
m \left| \rho_ {j, r, i} ^ {(t)} \right| \geq \gamma \sum_ {r = 1} ^ {m} \left| \rho_ {j, r, i} ^ {(t)} \right|, \tag {C.41}
$$
$$
\sum_ {r = 1} ^ {m} | \rho_ {j ^ {\prime}, r, i ^ {\prime}} ^ {(t)} | \geq m \gamma | \rho_ {j ^ {\prime}, r ^ {\prime}, i ^ {\prime}} ^ {(t)} |.
$$
Plugging (C.41) back into (C.8) completes the proof.
Lemma C.5. Let  $T_{1}$  be defined in Lemma C.2. Every neuron will never change its activation pattern after time  $T_{1}$ , i.e.,
$$
\operatorname {s i g n} (\langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \rangle) = \operatorname {s i g n} (\langle \mathbf {w} _ {j, r} ^ {(T _ {1})}, \mathbf {x} _ {i} \rangle),
$$
for any  $t \geq T_1$ ,  $j \in \{\pm 1\}$  and  $r \in [m]$ . Moreover, it holds that
$$
\operatorname {s i g n} \left(\left\langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \right\rangle\right) = j y _ {i}, \tag {C.42}
$$
for any  $t\geq T_1$ $j\in \{\pm 1\}$  and  $r\in [m]$
Proof of Lemma C.5. For  $j = y_{i}$  and  $t \geq 0$ , we have  $\underline{\rho}_{j,r,i}^{(t)} = 0$ , and so
$$
\begin{array}{l} \langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \rangle = \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \sum_ {i ^ {\prime} = 1} ^ {n} \rho_ {j, r, i ^ {\prime}} ^ {(t)} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle \\ = \langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \rangle + \overline {{\rho}} _ {j, r, i} ^ {(t)} + \sum_ {i ^ {\prime} \neq i} \rho_ {j, r, i ^ {\prime}} ^ {(t)} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle \\ \geq \overline {{\rho}} _ {j, r, i} ^ {(t)} - \sum_ {i ^ {\prime} \neq i} | \rho_ {j, r, i ^ {\prime}} ^ {(t)} | R _ {\mathrm {m i n}} ^ {- 2} p - \beta \\ \end{array}
$$
$$
\begin{array}{l} \geq \bar {\rho} _ {j, r, i} ^ {(t)} - \gamma^ {- 4} c _ {1} ^ {- 1} \bar {\rho} _ {j, r, i} ^ {(t)} R _ {\min } ^ {- 2} p n - \beta \\ = \left(1 - \gamma^ {- 4} c _ {1} ^ {- 1} R _ {\min } ^ {- 2} p n\right) \cdot \bar {\rho} _ {j, r, i} ^ {(t)} - \beta , \\ \end{array}
$$
where the first inequality is by triangle inequality; the second inequality is by  $|\rho_{j,r,i'}^{(t)}| \leq \gamma^{-4}c_1^{-1}\overline{\rho}_{y_i,r,i}^{(t)}$  from Lemma C.1 and Lemma C.4.
By (C.13), we have for  $t \geq T_1$  that
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(t)} \geq \frac {\widetilde {c} \gamma \eta R _ {\min } ^ {2} T _ {1}}{n m} = C ^ {\prime} \widetilde {c} \gamma R _ {\min } ^ {2} R _ {\max } ^ {- 2}. \tag {C.43}
$$
Therefore, by (C.5), (C.6) and (C.43), we know that
$$
(1 - \gamma^ {- 1} c _ {1} ^ {- 4} R _ {\min } ^ {- 2} p n) \cdot \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} > \beta , \forall r \in [ m ], i \in [ n ].
$$
and thus  $\mathrm{sign}(\langle \mathbf{w}_{j,r}^{(t)},\mathbf{x}_i\rangle) = 1$  for any  $r\in [m],i\in [n],j = y_{i}$
For  $j\neq y_{i}$  and any  $t\geq 0$  , we have  $\overline{\rho}_{j,r,i}^{(t)} = 0$  , and so
$$
\begin{array}{l} \left\langle \mathbf {w} _ {j, r} ^ {(t)}, \mathbf {x} _ {i} \right\rangle = \left\langle \mathbf {w} _ {j, r} ^ {(0)}, \mathbf {x} _ {i} \right\rangle + \sum_ {i ^ {\prime} = 1} ^ {n} \rho_ {j, r, i ^ {\prime}} ^ {(t)} \left\| \mathbf {x} _ {i ^ {\prime}} \right\| _ {2} ^ {- 2} \cdot \left\langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \right\rangle \\ = \underline {{\rho}} _ {j, r, i} ^ {(t)} + \sum_ {i ^ {\prime} \neq i} \rho_ {j, r, i ^ {\prime}} ^ {(t)} \| \mathbf {x} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \mathbf {x} _ {i ^ {\prime}}, \mathbf {x} _ {i} \rangle \\ \leq \underline {{\rho}} _ {j, r, i} ^ {(t)} + \sum_ {i ^ {\prime} \neq i} | \rho_ {j, r, i ^ {\prime}} ^ {(t)} | R _ {\min } ^ {- 2} p + \beta \\ \leq \underline {{\rho}} _ {j, r, i} ^ {(t)} - \gamma^ {- 1} c _ {2} ^ {- 1} \underline {{\rho}} _ {j, r, i} ^ {(t)} R _ {\min } ^ {- 2} p n - \beta \\ = (1 - \gamma^ {- 1} c _ {2} ^ {- 1} R _ {\min } ^ {- 2} p n) \underline {{\rho}} _ {j, r, i} ^ {(t)} - \beta , \\ \end{array}
$$
where the first inequality is by triangle inequality; the second inequality is by  $|\rho_{j,r,i'}^{(t)}| \leq \gamma^{-4}c_1^{-1}|\underline{\rho}_{-y_i,r,i'}^{(t)}|$  from Lemma C.1 and Lemma C.4.
By (C.13), we have
$$
\left| \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} \right| \geq \frac {\widetilde {c} \gamma \eta R _ {\min } ^ {2} T _ {1}}{n m} = C ^ {\prime} \widetilde {c} \gamma R _ {\min } ^ {2} R _ {\max } ^ {- 2}. \tag {C.44}
$$
Therefore, by (C.5), (C.6) and (C.44), we know that
$$
(1 - \gamma^ {- 4} c _ {1} ^ {- 1} R _ {\min } ^ {- 2} p n) \cdot | \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} | > \beta , \forall r \in [ m ], i \in [ n ],
$$
and thus  $\mathrm{sign}(\langle \mathbf{w}_{j,r}^{(t)},\mathbf{x}_i\rangle) = -1$  for  $j\neq y_{i}$ , which completes the proof.
D Stable Rank of Leaky ReLU Network
In this section, we consider the properties of stable rank of the weight matrix  $\mathbf{W}^{(t)}$  found by gradient descent at time  $t$ , defined as  $\| \mathbf{W}^{(t)} \|_F^2 / \| \mathbf{W}^{(t)} \|_2^2$ , the square of the ratio of the Frobenius norm to the spectral norm of  $\mathbf{W}^{(t)}$ . Given Lemma C.5, we have following coefficient update rule for  $t \geq T_1$  where  $T_1$  is defined in Lemma C.2:
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(t + 1)} = \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + \frac {\eta}{n m} \cdot \left| \ell_ {i} ^ {\prime (t)} \right| \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \tag {D.1}
$$
$$
\underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t + 1)} = \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} - \frac {\gamma \eta}{n m} \cdot \left| \ell_ {i} ^ {\prime (t)} \right| \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \tag {D.2}
$$
$$
| \ell_ {i} ^ {\prime (t)} | = \frac {1}{1 + \exp \{F _ {y _ {i}} (\mathbf {W} _ {y _ {i}} ^ {(t)} , \mathbf {x} _ {i}) - F _ {- y _ {i}} (\mathbf {W} _ {- y _ {i}} ^ {(t)} , \mathbf {x} _ {i}) \}}.
$$
Based on (D.1) and (D.2), we first introduce the following helpful lemmas.
Lemma D.1. Let  $T_{1}$  be defined in Lemma C.3. For any  $r, r' \in [m]$ ,  $i \in [n]$  and  $t \leq T_{1}$ ,
$$
\left| \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} - \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} \right| \leq C ^ {\prime}, \left| \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} - \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(t)} \right| \leq C ^ {\prime}. \tag {D.3}
$$
Proof of Lemma D.1. By (C.14), we can get
$$
| \rho_ {j, r, i} ^ {(t)} | \leq \frac {\eta R _ {\mathrm {m a x}} ^ {2} T _ {1}}{n m} = C ^ {\prime},
$$
for  $t \leq T_1$ . Notice that
$$
| \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} - \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} | \leq \max  \{| \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} |, | \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} | \},
$$
$$
\left. \left| \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} - \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(t)} \right| \leq \max  \left\{\left| \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} \right|, \left| \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(t)} \right| \right\}, \right.
$$
Lemma D.2. Let  $T_{1}$  be defined in Lemma C.3. For any  $r, r' \in [m]$ ,  $i \in [n]$  and  $t \geq T_{1}$ ,
$$
| \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} - \bar {\rho} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} | \leq C ^ {\prime}, | \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} - \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(t)} | \leq C ^ {\prime}.
$$
Proof of Lemma D.2. By (D.1) and (D.2), we can get for any  $r \in [m]$ ,  $i \in [n]$  and  $t \geq T_1$  that
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} = \overline {{\rho}} _ {y _ {i}, r, i} ^ {(T _ {1})} + \frac {\eta}{n m} \sum_ {s = T _ {1}} ^ {t - 1} | \ell_ {i} ^ {\prime (t)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2},
$$
$$
\underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} = \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(T _ {1})} + \frac {\eta}{n m} \sum_ {s = T _ {1}} ^ {t - 1} | \ell_ {i} ^ {\prime (t)} | \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}.
$$
Since  $\overline{\rho}_{y_i,r,i}^{(t)},\overline{\rho}_{y_i,r',i}^{(t)}$  possess the same increment and  $\underline{\rho}_{-y_i,r,i}^{(t)},\underline{\rho}_{-y_i,r',i}^{(t)}$  possess the same increment, we have
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} = \overline {{\rho}} _ {y _ {i}, r, i} ^ {(T _ {1})} - \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(T _ {1})},
$$
$$
\underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} - \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(t)} = \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(T _ {1})} - \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(T _ {1})}.
$$
$$
\max _ {i, r, r ^ {\prime}} \{| \overline {{\rho}} _ {y _ {i}, r, i} ^ {(T _ {1})} - \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(T _ {1})} |, | \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(T _ {1})} - \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(T _ {1})} | \} \leq \max _ {i, r, r ^ {\prime}} \{| \overline {{\rho}} _ {y _ {i}, r, i} ^ {(T _ {1})} |, | \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(T _ {1})} | \} \leq C ^ {\prime},
$$
Lemma D.3. Let  $T_{1}$  be defined in Lemma C.3. For  $t \geq T_{1}$ , it holds that
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \overline {{\rho}} _ {y _ {i}, r, i} ^ {(T _ {1})} = \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(t)} - \overline {{\rho}} _ {y _ {i}, r ^ {\prime}, i} ^ {(T _ {1})},
$$
$$
\underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} - \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(T _ {1})} = \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(t)} - \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(T _ {1})},
$$
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \overline {{\rho}} _ {y _ {i}, r, i} ^ {(T _ {1})} = \big (\left| \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(t)} \right| - \left| \underline {{\rho}} _ {- y _ {i}, r ^ {\prime}, i} ^ {(T _ {1})} \right| \big) / \gamma ,
$$
Proof of Lemma D.3. By Lemma C.3 about the activation pattern after time  $T_{1}$ , we can get
$$
\bar {\rho} _ {y _ {i}, r, i} ^ {(t + 1)} = \bar {\rho} _ {y _ {i}, r, i} ^ {(t)} + \frac {\eta}{n m} \cdot \left| \ell_ {i} ^ {\prime (t)} \right| \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \tag {D.4}
$$
$$
\underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t + 1)} = \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} - \frac {\gamma \eta}{n m} \cdot \left| \ell_ {i} ^ {\prime (t)} \right| \cdot \| \mathbf {x} _ {i} \| _ {2} ^ {2}, \tag {D.5}
$$
for  $t \geq T_1$ . Recursively using (D.4) and (D.5)  $t - T_1$  times, we can get
$$
\overline {{\rho}} _ {y _ {i}, r, i} ^ {(t)} - \overline {{\rho}} _ {y _ {i}, r, i} ^ {(T _ {1})} = \frac {\eta \| \mathbf {x} _ {i} \| _ {2} ^ {2}}{n m} \sum_ {s = T _ {1}} ^ {t - 1} | \ell_ {i} ^ {\prime (s)} |,
$$
$$
| \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(t)} | - | \underline {{\rho}} _ {- y _ {i}, r, i} ^ {(T _ {1})} | = \frac {\gamma \eta \| \mathbf {X} _ {i} \| _ {2} ^ {2}}{n m} \sum_ {s = T _ {1}} ^ {t - 1} | \ell_ {i} ^ {\prime (s)} |.
$$
This indicates that for different  $r, r' \in [m]$ ,  $\overline{\rho}_{y_i,r,i}^{(t)} - \overline{\rho}_{y_i,r,i}^{(T_1)}$  and  $\overline{\rho}_{y_i,r',i}^{(t)} - \overline{\rho}_{y_i,r',i}^{(T_1)}$  are the same, whereas  $\gamma (|\underline{\rho}_{-y_i,r,i}^{(t)}| - |\underline{\rho}_{-y_i,r,i}^{(T_1)}|)$  and  $\overline{\rho}_{y_i,r',i}^{(t)} - \overline{\rho}_{y_i,r',i}^{(T_1)}$  are the same, which completes the proof.
Now we are ready to prove the second bullet of Theorem 4.1.
Lemma D.4. Throughout the gradient descent trajectory, the stable rank of the weights  $\mathbf{W}_j$  satisfies,
$$
\lim _ {t \to \infty} \frac {\| \mathbf {W} _ {j} \| _ {F} ^ {2}}{\| \mathbf {W} _ {j} \| _ {2} ^ {2}} = 1, \forall j \in \{\pm 1 \},
$$
with a decreasing rate of  $O\big(1 / \log (t)\big)$ .
Proof of Lemma D.4. By Definition 5.1, we have