SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\Pr \left[ i \in S _ {D _ {j}, S u r e} \right] \geq \Theta \left(\frac {1}{\sqrt {\tau \log \log d} \sigma_ {B} \cdot (\log d) ^ {\frac {1 8 \tau \sigma_ {B} ^ {2}}{2 5}}}\right). \tag {339}
$$
Lemma E.15 (Mixture of Gaussians in [99]: Activation Pattern). Assume the same conditions as in Lemma E.13, for all  $j \in [4]$ ,  $i \in S_{D_j, S_{\text{ure}}}$ , we have
(1) When  $\mathbf{x} \sim \mathcal{N}_j(\mu_j, \sigma_B I_{d \times d})$ , the activation probability satisfies,
$$
\Pr_ {\mathbf {x} \sim \mathcal {N} _ {j} \left(\mu_ {j}, \sigma_ {B} I _ {d \times d}\right)} \left[ \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle - \mathbf {b} _ {i} \geq 0 \right] \geq 1 - \frac {1}{\left(\log d\right) ^ {\frac {\tau}{5 0}}}. \tag {340}
$$
(2) For all  $j^{\prime}\neq j,j^{\prime}\in [4]$ , when  $\mathbf{x}\sim \mathcal{N}_{j^{\prime}}(\mu_{j^{\prime}},\sigma_{B}I_{d\times d})$ , the activation probability satisfies,
$$
\Pr_ {\mathbf {x} \sim \mathcal {N} _ {j ^ {\prime}} \left(\mu_ {j ^ {\prime}}, \sigma_ {B} I _ {d \times d}\right)} \left[ \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle - \mathbf {b} _ {i} \geq 0 \right] \leq O \left(\frac {1}{\left(\log d\right) ^ {\frac {\tau}{2}}}\right). \tag {341}
$$
Proof of Lemma E.15. In the proof, we need  $\tilde{b} = \sqrt{\tau d\log\log d}\sigma_w\sigma_B$ . For the first statement, when  $\mathbf{x}\sim \mathcal{N}_j(\mu_j,\sigma_B I_{d\times d})$ , by  $C_{\text{Sure},1}\geq \frac{6}{5}$ , we have
$$
\begin{array}{l} \Pr_ {\mathbf {x} \sim \mathcal {N} _ {j} \left(\mu_ {j}, \sigma_ {B} I _ {d \times d}\right)} \left[ \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle - \mathbf {b} _ {i} \geq 0 \right] \geq \Pr_ {\mathbf {x} \sim \mathcal {N} \left(0, \sigma_ {B} I _ {d \times d}\right)} \left[ \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle \geq (1 - C _ {S u r e, 1}) \mathbf {b} _ {i} \right] (342) \\ \geq \Pr_ {\mathbf {x} \sim \mathcal {N} \left(0, \sigma_ {B} I _ {d \times d}\right)} \left[ \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle \geq - \frac {\mathbf {b} _ {i}}{5} \right] (343) \\ = 1 - \Pr_ {\mathbf {x} \sim \mathcal {N} (0, \sigma_ {B} I _ {d \times d})} \left[ \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle \leq - \frac {\mathbf {b} _ {i}}{5} \right] (344) \\ \geq 1 - \exp \left(- \frac {\mathbf {b} _ {i} ^ {2}}{5 0 d \sigma_ {w} ^ {2} \sigma_ {B} ^ {2}}\right) (345) \\ \geq 1 - \frac {1}{(\log d) ^ {\frac {\tau}{5 0}}}, (346) \\ \end{array}
$$
For the second statement, we prove similarly by  $0 < C_{\text{Sure},2} \leq \frac{\sqrt{2}}{\sqrt{\tau \log \log d}}$ .
Then, Lemma E.16 gives gradients of neurons in  $S_{D_j, \text{Sure}}$ . It shows that these gradients are highly aligned with  $D_j$ .
Lemma E.16 (Mixture of Gaussians in [99]: Feature Emergence). Assume the same conditions as in Lemma E.13, for all  $j \in [4]$ ,  $i \in S_{D_j, S_{\text{ure}}}$ , we have
$$
\left\langle \mathbb {E} _ {(\mathbf {x}, y)} \left[ y \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle - \mathbf {b} _ {i}\right) \mathbf {x} \right], y _ {(j)} D _ {j} \right\rangle \tag {347}
$$
$$
\geq \frac {1}{4} \sqrt {d} \left(1 - O \left(\frac {1}{\left(\log d\right) ^ {\frac {\tau}{5 0}}}\right)\right) - \sigma_ {B} O \left(\frac {1}{\left(\log d\right) ^ {0 . 0 1 8 \tau}}\right). \tag {348}
$$
$$
\left| \left\langle \mathbb {E} _ {(\mathbf {x}, y)} \left[ y \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle - \mathbf {b} _ {i}\right) \mathbf {x} \right], D _ {j} ^ {\perp} \right\rangle \right| \leq \sigma_ {B} O \left(\frac {1}{(\log d) ^ {0 . 0 1 8 \tau}}\right). \tag {349}
$$
Proof of Lemma E.16. For all  $j \in [4]$ ,  $i \in S_{D_j, Sure}$ , we have
$$
\begin{array}{l} \mathbb {E} _ {(\mathbf {x}, y)} \left[ y \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle - \mathbf {b} _ {i}\right) \mathbf {x} \right] (350) \\ = \sum_ {l \in [ 4 ]} \frac {1}{4} \mathbb {E} _ {\mathbf {x} \sim \mathcal {N} _ {l} (\mathbf {x})} \left[ y \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle - \mathbf {b} _ {i}\right) \mathbf {x} \right] (351) \\ = \sum_ {l \in [ 4 ]} \frac {1}{4} y _ {(l)} \mathbb {E} _ {\mathbf {x} \sim \mathcal {N} (0, \sigma_ {l} I _ {d \times d})} \left[ \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} + \mu_ {l} \right\rangle - \mathbf {b} _ {i}\right) (\mathbf {x} + \mu_ {l}) \right]. (352) \\ \end{array}
$$
Thus, by Lemma F.7 and Lemma E.15,
For any unit vector  $D_{j}^{\perp}$  which is orthogonal with  $D_{j}$ , similarly, we have
$$
\begin{array}{l} \left| \left\langle \mathbb {E} _ {(\mathbf {x}, y)} \left[ y \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} \right\rangle - \mathbf {b} _ {i}\right) \mathbf {x} \right], D _ {j} ^ {\perp} \right\rangle \right| (362) \\ \leq \frac {1}{4} \left| \mathbb {E} _ {\mathbf {x} \sim \mathcal {N} (0, \sigma_ {B} I)} \left[ \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} + \mu_ {j} \right\rangle - \mathbf {b} _ {i}\right) \mathbf {x} ^ {\top} D _ {j} ^ {\perp} \right] \right| (363) \\ + \sum_ {l \in [ 4 ], l \neq j} \frac {1}{4} \left| \mathbb {E} _ {\mathbf {x} \sim \mathcal {N} (0, \sigma_ {l} I)} \left[ \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} + \mu_ {l} \right\rangle - \mathbf {b} _ {i}\right) (\mathbf {x} + \mu_ {l}) ^ {\top} D _ {j} ^ {\perp} \right] \right| (364) \\ \leq \frac {1}{4} \left| \mathbb {E} _ {\mathbf {x} \sim \mathcal {N} (0, \sigma_ {B} I)} \left[ \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} + \mu_ {j} \right\rangle - \mathbf {b} _ {i}\right) \mathbf {x} ^ {\top} D _ {j} ^ {\perp} \right] \right| (365) \\ + \sum_ {l \in [ 4 ], l \neq j} \frac {1}{4} \left| \mathbb {E} _ {\mathbf {x} \sim \mathcal {N} (0, \sigma_ {l} I)} \left[ \sigma^ {\prime} \left(\left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {x} + \mu_ {l} \right\rangle - \mathbf {b} _ {i}\right) \mathbf {x} ^ {\top} D _ {j} ^ {\perp} \right] \right| (366) \\ \leq \sigma_ {B} O \left(\frac {1}{(\log d) ^ {0 . 0 1 8 \tau}}\right). (367) \\ \end{array}
$$
E.3.2 Mixture of Gaussians - XOR: Final Guarantee
Lemma E.17 (Mixture of Gaussians in [99]: Existence of Good Networks). Assume the same conditions as in Lemma E.13 and let  $\tau = 1$  and when  $0 < \tilde{\tau} \leq O\left(\frac{d}{\sigma_B^2\log d}\right)$ . Define
$$
f ^ {*} (\mathbf {x}) = \sum_ {j = 1} ^ {4} \frac {y _ {(j)}}{\sqrt {\tilde {\tau} \log d} \sigma_ {B}} \left[ \sigma \left(\left\langle D _ {j}, \mathbf {x} \right\rangle - 2 \sqrt {\tilde {\tau} \log d} \sigma_ {B}\right) \right]. \tag {368}
$$
For  $\mathcal{D}_{\text{mixture - xor}}$  setting, we have  $f^{*} \in \mathcal{F}_{d,r,B_{F},S_{p,\gamma ,B_{G}}}$ , where  $B_{F} = (B_{a1},B_{a2},B_{b}) = \left(\frac{1}{\sqrt{\tilde{\tau}\log d}\sigma_{B}},\frac{2}{\sqrt{\tilde{\tau}\log d}\sigma_{B}},2\sqrt{\tilde{\tau}\log d}\sigma_{B}\right)$ ,  $p = \Omega \left(\frac{1}{\sigma_B\cdot(\log d)^{\sigma_B^2}}\right)$ ,  $\gamma = \frac{\sigma_B}{\sqrt{d}}$ ,  $r = 4$ ,  $B_{G} = \frac{1}{5}\sqrt{d}$  and  $B_{x1} = (1 + \sigma_B)\sqrt{d}$ ,  $B_{x2} = (1 + \sigma_B)^2 d$ . We also have  $\mathrm{OPT}_{d,r,B_F,S_{p,\gamma ,B_G}} \leq \frac{3}{d^{\bar{\tau}}} +\frac{4}{d^{0.9\bar{\tau} - 1}\sqrt{\tilde{\tau}\log d}}$ .
Proof of Lemma E.17. We finish the proof by following the proof of Lemma E.11
Theorem E.18 (Mixture of Gaussians in [99]: Main Result). For  $\mathcal{D}_{\text{mixture - xor}}$  setting with Assumption E.12, when  $d$  is large enough, for any  $\delta \in (0, 1)$  and for any  $\epsilon \in (0, 1)$  when
$$
m = \Omega \left(\sigma_ {B} (\log d) ^ {\sigma_ {B} ^ {2}} \left(\left(\log \left(\frac {1}{\bar {\delta}}\right)\right) ^ {2} + \frac {1 + \sigma_ {B}}{\epsilon^ {4}}\right) + \frac {1}{\sqrt {\delta}}\right) \leq e ^ {d}, \tag {369}
$$
$$
T = \operatorname {p o l y} \left(\sigma_ {B}, 1 / \epsilon , 1 / \delta , \log d\right), \tag {370}
$$
$$
n = \tilde {\Omega} \left(\frac {m ^ {3} \left(1 + \sigma_ {B} ^ {2}\right)}{\epsilon^ {2} \max  \left\{\sigma_ {B} \cdot (\log d) ^ {\sigma_ {B} ^ {2}}, 1 \right\}} + \sigma_ {B} \cdot (\log d) ^ {\sigma_ {B} ^ {2}} + \frac {T m}{\delta}\right), \tag {371}
$$
trained by Algorithm 1 with hinge loss, with probability at least  $1 - \delta$  over the initialization and training samples, with proper hyper-parameters, there exists  $t\in [T]$  such that
$$
\Pr \left[ \operatorname {s i g n} \left(f _ {\Xi^ {(t)}} (\mathbf {x})\right) \neq y \right] \leq O \left(\left(1 + \sigma_ {B} ^ {\frac {3}{2}}\right) \left(\frac {1}{d ^ {\frac {1}{4}}} + \frac {\left(\log n\right) ^ {\frac {1}{4}}}{n ^ {\frac {1}{4}}}\right)\right) + \epsilon . \tag {372}
$$
Proof of Theorem E.18. Let  $\tilde{b} = \sqrt{d\log\log d}\sigma_w\sigma_B$ . By Lemma E.17, let  $\tau = 1$  and when  $\tilde{\tau} = O\left(\frac{d}{\sigma_B^2\log d}\right)$ , we have  $f^{*} \in \mathcal{F}_{d,r,B_F,S_p,\gamma,B_G}$ , where  $B_F = (B_{a1},B_{a2},B_b) = \left(\frac{1}{\sqrt{\tau}\log d\sigma_B},\frac{2}{\sqrt{\tau}\log d\sigma_B},2\sqrt{\tilde{\tau}}\log d\sigma_B\right)$ ,  $p = \Omega\left(\frac{1}{\sigma_B\cdot(\log d)^{\sigma_B^2}}\right)$ ,  $\gamma = \frac{\sigma_B}{\sqrt{d}}$ ,  $r = 4$ ,  $B_G = \frac{1}{5}\sqrt{d}$  and  $B_{x1} = (1 + \sigma_B)\sqrt{d}$ ,  $B_{x2} = (1 + \sigma_B)^2 d$ . We also have  $\mathrm{OPT}_{d,r,B_F,S_p,\gamma,B_G} \leq \frac{3}{d^{\tilde{\tau}}} + \frac{4}{d^{0.9\tilde{\tau} - 1}\sqrt{\tilde{\tau}\log d}}$ . Adjust  $\sigma_w$  such that  $\tilde{b} = \sqrt{d\log\log d}\sigma_w\sigma_B = \Theta\left(\frac{B_G^{\frac{1}{4}}B_{a2}B_b^{\frac{3}{4}}}{\sqrt{rB_{a1}}}\right)$ . Injecting above parameters into Theorem 3.12, we have with probability at least  $1 - \delta$  over the initialization, with proper hyperparameters, there exists  $t \in [T]$  such that
$$
\Pr \left[ \operatorname {s i g n} \left(f _ {\Xi^ {(t)}} (\mathbf {x})\right) \neq y \right] \leq O \left(\left(1 + \sigma_ {B} ^ {\frac {3}{2}}\right) \left(\frac {1}{d ^ {\frac {1}{4}}} + \frac {\left(\log n\right) ^ {\frac {1}{4}}}{n ^ {\frac {1}{4}}}\right)\right) + \epsilon . \tag {373}
$$
E.4 Parity Functions
We recap the problem setup in Section 4.2 for readers' convenience.
E.4.1 Problem Setup
Data Distributions. Suppose  $\mathbf{M} \in \mathbb{R}^{d \times D}$  is an unknown dictionary with  $D$  columns that can be regarded as patterns. For simplicity, assume  $d = D$  and  $\mathbf{M}$  is orthonormal. Let  $\phi \in \mathbb{R}^d$  be a hidden representation vector. Let  $A \subseteq [D]$  be a subset of size  $rk$  corresponding to the class relevant patterns
and  $r$  is an odd number. Then the input is generated by  $\mathbf{M}\phi$ , and some function on  $\phi_A$  generates the label. WLOG, let  $A = \{1,\dots ,rk\}$ ,  $A^{\perp} = \{rk + 1,\dots ,d\}$ . Also, we split  $A$  such that for all  $j\in [r]$ ,  $A_{j} = \{(j - 1)k + 1,\dots ,jk\}$ . Then the input  $\mathbf{x}$  and the class label  $y$  are given by:
$$
\mathbf {x} = \mathbf {M} \phi , y = g ^ {*} \left(\phi_ {A}\right) = \operatorname {s i g n} \left(\sum_ {j = 1} ^ {r} \operatorname {X O R} \left(\phi_ {A _ {j}}\right)\right), \tag {374}
$$
where  $g^{*}$  is the ground-truth labeling function mapping from  $\mathbb{R}^{rk}$  to  $\mathcal{V} = \{\pm 1\}$ ,  $\phi_{A}$  is the sub-vector of  $\phi$  with indices in  $A$ , and  $\mathrm{XOR}(\phi_{A_j}) = \prod_{l\in A_j}\phi_l$  is the parity function.
We still need to specify the distribution of  $\phi$ , which determines the structure of the input distribution:
$$
\mathcal {X} := \left(1 - 2 r p _ {A}\right) \mathcal {X} _ {U} + \sum_ {j \in [ r ]} p _ {A} \left(\mathcal {X} _ {j, +} + \mathcal {X} _ {j, -}\right). \tag {375}
$$
For all corresponding  $\phi_{A^{\perp}}$  in  $\mathcal{X}$ , we have  $\forall l\in A^{\perp}$ , independently:
$$
\phi_ {l} = \left\{ \begin{array}{l l} + 1, & \text {w . p .} p _ {o} \\ - 1, & \text {w . p .} p _ {o} \\ 0, & \text {w . p .} 1 - 2 p _ {o} \end{array} \right.
$$
where  $p_{o}$  controls the signal noise ratio: if  $p_{o}$  is large, then there are many nonzero entries in  $A^{\perp}$  which are noise interfering with the learning of the ground-truth labeling function on  $A$ .
For corresponding  $\phi_A$ , any  $j \in [r]$ , we have
- In  $\mathcal{X}_{j, + }$ $\phi_{A_j} = [+1, + 1,\dots , + 1]^\top$  and  $\phi_{A\setminus A_j}$  only have zero elements.
- In  $\mathcal{X}_{j, - }$ $\phi_{A_j} = [-1, - 1,\dots , - 1]^{\top}$  and  $\phi_{A\setminus A_j}$  only have zero elements.
- In  $\mathcal{X}_U$ , we have  $\phi_A$  draw from  $\{+1, -1\}^{rk}$  uniformly.
We call this data distribution  $\mathcal{D}_{\text{parity}}$ .
Assumption E.19 (Parity Functions. Recap of Assumption 4.5). Let  $8 \leq \tau \leq d$  be a parameter that will control our final error guarantee. Assume  $k$  is an odd number and:
$$
k \geq \Omega (\tau \log d), \quad d \geq r k + \Omega (\tau r \log d), \quad p _ {o} = O \left(\frac {r k}{d - r k}\right), \quad p _ {A} \geq \frac {1}{d}. \tag {376}
$$
Remark E.20. The assumptions require  $k, d$ , and  $p_A$  to be sufficiently large so as to provide enough large signals for learning. When  $p_o = \Theta\left(\frac{rk}{d - rk}\right)$  means that the signal noise ratio is constant: the expected norm of  $\phi_A$  and that of  $\phi_{A^\perp}$  are comparable.
To apply our framework, again we only need to compute the parameters in the Gradient Feature set and the corresponding optimal approximation loss. To this end, we first define the gradient features: For all  $j \in [r]$ , let
$$
D _ {j} = \frac {\sum_ {l \in A _ {j}} \mathbf {M} _ {l}}{\| \sum_ {l \in A _ {j}} \mathbf {M} _ {l} \| _ {2}}. \tag {377}
$$
Remark E.21. Our data distribution is symmetric, which means for any  $\phi \in \mathbb{R}^d$ :
-  $-y = g^{*}(-\phi_{A})$  and  $-x = \mathbf{M}(-\phi)$ ,
$\mathbb{P}(\phi) = \mathbb{P}(-\phi),$
$\mathbb{E}_{(\mathbf{x},y)}[y\mathbf{x}] = \mathbf{0}.$
Definition E.22 (Parity Functions: Subset of Nice Gradients Set). Recall  $\mathbf{w}_i^{(0)}$  is the weight for the  $i$ -th neuron at initialization. For all  $j \in [r]$ , let  $S_{D_j, S_\text{ure}} \subseteq [m]$  be those neurons that satisfy
$\left\langle \mathbf{w}_i^{(0)},D_j\right\rangle \geq \frac{C_{Sure,1}}{\sqrt{k}}\mathbf{b}_i,$
-  $\left|\left\langle \mathbf{w}_i^{(0)},D_{j'}\right\rangle \right|\leq \frac{C_{Sure,2}}{\sqrt{k}}\mathbf{b}_i,\text{for all} j'\neq j,j'\in [r],$
$\left\| P_A \mathbf{w}_i^{(0)} \right\|_2 \leq \Theta (\sqrt{rk} \sigma_w),$
$\left\| P_{A^{\perp}}\mathbf{w}_i^{(0)}\right\| _2\leq \Theta (\sqrt{d - rk}\sigma_w),$
where  $P_A, P_{A^\perp}$  are the projection operator on the space  $\mathbf{M}_A$  and  $\mathbf{M}_{A^\perp}$ .
E.4.2 Parity Functions: Feature Learning
We show the important Lemma E.23 first and defer other Lemmas after it.
Lemma E.23 (Parity Functions: Gradient Feature Set. Part statement of Lemma 4.7). Let  $C_{\text{Sure},1} = \frac{3}{2}$ ,  $C_{\text{Sure},2} = \frac{1}{2}$ ,  $\tilde{b} = C_b \sqrt{\tau r k \log d} \sigma_w$ , where  $C_b$  is a large enough universal constant. For  $\mathcal{D}_{\text{parity}}$  setting, we have  $(D_j, +1)$ ,  $(D_j, -1) \in S_{p,\gamma,B_G}$  for all  $j \in [r]$ , where
$$
p = \Theta \left(\frac {1}{\sqrt {\tau r \log d} \cdot d ^ {(9 C _ {b} ^ {2} \tau r / 8)}}\right), \quad \gamma = \frac {1}{d ^ {\tau - 2}}, \quad B _ {G} = \sqrt {k} p _ {A} - O \left(\frac {\sqrt {k}}{d ^ {\tau}}\right). \tag {378}
$$
Proof of Lemma E.23. Note that for all  $l \in [d]$ , we have  $\mathbf{M}_l^\top \mathbf{x} = \phi_l$ . For all  $j \in [r]$ , by Lemma E.26, for all  $i \in S_{D_j, Sure}$ , when  $\gamma = \frac{1}{d\tau^{-2}}$
$$
\begin{array}{l} \left| \left\langle G \left(\mathbf {w} _ {i} ^ {(0)}, \mathbf {b} _ {i}\right), D _ {j} \right\rangle \right| - (1 - \gamma) \| G \left(\mathbf {w} _ {i} ^ {(0)}, \mathbf {b} _ {i}\right) \| _ {2} (379) \\ = \left| \left\langle G \left(\mathbf {w} _ {i} ^ {(0)}, \mathbf {b} _ {i}\right), \frac {\sum_ {l \in A _ {j}} \mathbf {M} _ {l}}{\sqrt {k}} \right\rangle \right| - (1 - \gamma) \| G \left(\mathbf {w} _ {i} ^ {(0)}, \mathbf {b} _ {i}\right) \| _ {2} (380) \\ \geq \sqrt {k} p _ {A} - O \left(\frac {\sqrt {k}}{d ^ {\tau}}\right) - \left(1 - \frac {1}{d ^ {\tau - 2}}\right) \sqrt {k p _ {A} ^ {2} + \sum_ {l \in [ d ]} O \left(\frac {1}{d ^ {\tau}}\right) ^ {2}} (381) \\ \geq \sqrt {k} p _ {A} - O \left(\frac {\sqrt {k}}{d ^ {\tau}}\right) - \left(1 - \frac {1}{d ^ {\tau - 2}}\right) \left(\sqrt {k} p _ {A} + O \left(\frac {1}{d ^ {\tau - \frac {1}{2}}}\right)\right) (382) \\ \geq \frac {\sqrt {k} p _ {A}}{d ^ {\tau - 2}} - O \left(\frac {\sqrt {k}}{d ^ {\tau}}\right) - O \left(\frac {1}{d ^ {\tau - \frac {1}{2}}}\right) (383) \\ > 0. (384) \\ \end{array}
$$
Thus, we have  $G(\mathbf{w}_i^{(0)}, \mathbf{b}_i) \in \mathcal{C}_{D_j,\gamma}$  and  $\sqrt{k} p_A - O\left(\frac{\sqrt{k}}{d^\tau}\right) \leq \| G(\mathbf{w}_i^{(0)}, \mathbf{b}_i)\|_2 \leq \sqrt{k} p_A + O\left(\frac{1}{d^{\tau - \frac{1}{2}}}\right)$ ,  $\frac{\mathbf{b}_i}{|\mathbf{b}_i|} = +1$ . Thus, by Lemma E.24, we have
$$
\begin{array}{l} \Pr_ {\mathbf {w}, b} \left[ G (\mathbf {w}, b) \in \mathcal {C} _ {D _ {j}, \gamma} \text {a n d} \| G (\mathbf {w}, b) \| _ {2} \geq B _ {G} \text {a n d} \frac {b}{| b |} = + 1 \right] (385) \\ \geq \Pr [ i \in S _ {D _ {j}, S u r e} ] (386) \\ \geq p. (387) \\ \end{array}
$$
Thus,  $(D_{j}, + 1)\in S_{p,\gamma ,B_{G}}$ . Since  $\mathbb{E}_{(\mathbf{x},y)}[y\mathbf{x}] = \mathbf{0}$ , by Lemma F.2 and considering  $i\in [2m]\setminus [m]$  we have  $(D_j, - 1)\in S_{p,\gamma ,B_G}$ . We finish the proof.
Below are Lemmas used in the proof of Lemma E.23. In Lemma E.24, we calculate  $p$  used in  $S_{p,\gamma,B_G}$ .
Lemma E.24 (Parity Functions: Geometry at Initialization. Lemma B.2 in [7]). Assume the same conditions as in Lemma E.23, recall for all  $i \in [m]$ ,  $\mathbf{w}_i^{(0)} \sim \mathcal{N}(0, \sigma_w^2 I_{d \times d})$ , over the random initialization, we have for all  $i \in [m], j \in [r]$
$$
\Pr \left[ i \in S _ {D _ {j}, S u r e} \right] \geq \Theta \left(\frac {1}{\sqrt {\tau r \log d} \cdot d ^ {\left(9 C _ {b} ^ {2} \tau r / 8\right)}}\right). \tag {388}
$$
Proof of Lemma E.24. For every  $i\in [m],j,j'\in [r],j\neq j'$  , by Lemma F.6,
$$
p _ {1} = \Pr \left[ \left\langle \mathbf {w} _ {i} ^ {(0)}, D _ {j} \right\rangle \geq \frac {C _ {\text {S u r e} , 1}}{\sqrt {k}} \mathbf {b} _ {i} \right] = \Theta \left(\frac {1}{\sqrt {\tau r \log d} \cdot d ^ {\left(9 C _ {b} ^ {2} \tau r / 8\right)}}\right) \tag {389}
$$
$$
p _ {2} = \Pr \left[ \left| \left\langle \mathbf {w} _ {i} ^ {(0)}, D _ {j ^ {\prime}} \right\rangle \right| \geq \frac {C _ {S u r e , 2}}{\sqrt {k}} \mathbf {b} _ {i} \right] = \Theta \left(\frac {1}{\sqrt {\tau r \log d} \cdot d ^ {\left(C _ {b} ^ {2} \tau r / 8\right)}}\right). \tag {390}
$$
On the other hand, if  $X$  is a  $\chi^2(k)$  random variable, by Lemma F.5, we have
$$
\Pr (X \geq k + 2 \sqrt {k x} + 2 x) \leq e ^ {- x}. \tag {391}
$$
Therefore, by assumption  $rk \geq \Omega(\tau r \log d), d - rk \geq \Omega(\tau r \log d)$ , we have
$$
\Pr \left(\frac {1}{\sigma_ {w} ^ {2}} \left\| P _ {A} \mathbf {w} _ {i} ^ {(0)} \right\| _ {2} ^ {2} \geq r k + 2 \sqrt {\left(9 C _ {b} ^ {2} \tau r / 8 + 2\right) r k \log d} + 2 \left(9 C _ {b} ^ {2} \tau r / 8 + 2\right) \log d\right) \tag {392}
$$
$$
\leq O \left(\frac {1}{d ^ {2} \cdot d ^ {\left(9 C _ {b} ^ {2} \tau r / 8\right)}}\right), \tag {393}
$$
$$
\Pr \left(\frac {1}{\sigma_ {w} ^ {2}} \left\| P _ {A} \mathbf {w} _ {i} ^ {(0)} \right\| _ {2} ^ {2} \geq (d - r k) + 2 \sqrt {(9 C _ {b} ^ {2} \tau r / 8 + 2) (d - r k) \log d} + 2 (9 C _ {b} ^ {2} \tau r / 8 + 2) \log d\right)
$$
$$
\leq O \left(\frac {1}{d ^ {2} \cdot d ^ {\left(9 C _ {b} ^ {2} \tau r / 8\right)}}\right). \tag {394}
$$
Thus, by union bound, and  $D_{1},\ldots ,D_{r}$  being orthogonal with each other, we have
$$
\begin{array}{l} \Pr \left[ i \in S _ {D _ {j}, S u r e} \right] \geq p _ {1} (1 - p _ {2}) ^ {r - 1} - O \left(\frac {1}{d ^ {2} \cdot d ^ {\left(9 C _ {b} ^ {2} \tau r / 8\right)}}\right) (395) \\ = \Theta \left(\frac {1}{\sqrt {\tau r \log d} \cdot d ^ {\left(9 C _ {b} ^ {2} \tau r / 8\right)}} \cdot \left(1 - \frac {r}{\sqrt {\tau r \log d} \cdot d ^ {\left(C _ {b} ^ {2} \tau r / 8\right)}}\right)\right) (396) \\ - O \left(\frac {1}{d ^ {2} \cdot d ^ {\left(9 C _ {b} ^ {2} \tau r / 8\right)}}\right) (397) \\ = \Theta \left(\frac {1}{\sqrt {\tau r \log d} \cdot d ^ {\left(9 C _ {b} ^ {2} \tau r / 8\right)}}\right). (398) \\ \end{array}
$$
In Lemma E.25, we compute the activation pattern for the neurons in  $S_{D_j, Sure}$ .
Lemma E.25 (Parity Functions: Activation Pattern). Assume the same conditions as in Lemma E.23, for all  $j \in [r]$ ,  $i \in S_{D_j, S_{\text{ure}}}$ , we have
(1) When  $\mathbf{x} \sim \mathcal{X}$ , we have
$$
\Pr_ {\mathbf {x} \sim \mathcal {X}} \left[ \left| \sum_ {l \in A ^ {\perp}} \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {M} _ {l} \phi_ {l} \right\rangle \right| \geq t \right] \leq \exp \left(- \frac {t ^ {2}}{\Theta (r k \sigma_ {w} ^ {2})}\right). \tag {399}
$$
(2) When  $\mathbf{x} \sim \mathcal{X}_U$ , we have
$$
\Pr_ {\mathbf {x} \sim \mathcal {X} _ {U}} \left[ \left| \sum_ {l \in A} \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {M} _ {l} \phi_ {l} \right\rangle \right| \geq t \right] \leq \exp \left(- \frac {t ^ {2}}{\Theta (r k \sigma_ {w} ^ {2})}\right). \tag {400}
$$
(3) When  $\mathbf{x} \sim \mathcal{X}_U$ , the activation probability satisfies,
$$
\Pr_ {\mathbf {x} \sim \mathcal {X} _ {U}} \left[ \sum_ {l \in [ d ]} \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {M} _ {l} \phi_ {l} \right\rangle - \mathbf {b} _ {i} \geq 0 \right] \leq O \left(\frac {1}{d ^ {\tau}}\right). \tag {401}
$$
(4) When  $\mathbf{x} \sim \mathcal{X}_{j, + }$ , the activation probability satisfies,
$$
\Pr_ {\mathbf {x} \sim \mathcal {X} _ {j, +}} \left[ \sum_ {l \in [ d ]} \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {M} _ {l} \phi_ {l} \right\rangle - \mathbf {b} _ {i} \geq 0 \right] \geq 1 - O \left(\frac {1}{d ^ {\tau}}\right). \tag {402}
$$
(5) For all  $j' \neq j, j' \in [r]$ ,  $s \in \{+, -\}$ , when  $\mathbf{x} \sim \mathcal{X}_{j',s}$ , or  $\mathbf{x} \sim \mathcal{X}_{j,-}$ , the activation probability satisfies,
$$
\Pr \left[ \sum_ {l \in [ d ]} \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {M} _ {l} \phi_ {l} \right\rangle - \mathbf {b} _ {i} \geq 0 \right] \leq O \left(\frac {1}{d ^ {\tau}}\right). \tag {403}
$$
Proof of Lemma E.25. For the first statement, when  $\mathbf{x} \sim \mathcal{X}$ , note that  $\left\langle \mathbf{w}_i^{(0)}, \mathbf{M}_l \right\rangle \phi_l$  is a mean-zero sub-Gaussian random variable with sub-Gaussian norm  $\Theta\left(\left|\left\langle \mathbf{w}_i^{(0)}, \mathbf{M}_l \right\rangle \right| \sqrt{p_o}\right)$ .
$$
\begin{array}{l} \Pr_ {\mathbf {x} \sim \mathcal {X}} \left[ \left| \sum_ {l \in A ^ {\perp}} \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {M} _ {l} \phi_ {l} \right\rangle \right| \geq t \right] = \Pr_ {\mathbf {x} \sim \mathcal {X}} \left[ \left| \sum_ {l \in A ^ {\perp}} \left\langle \mathbf {w} _ {i} ^ {(0)}, \mathbf {M} _ {l} \right\rangle \phi_ {l} \right| \geq t \right] (404) \\ \leq \exp \left(- \frac {t ^ {2}}{\sum_ {l \in A ^ {\perp}} \Theta \left(\left\langle \mathbf {w} _ {i} ^ {(0)} , \mathbf {M} _ {l} \right\rangle^ {2} p _ {o}\right)}\right) (405) \\ \leq \exp \left(- \frac {t ^ {2}}{\Theta \left((d - r k) \sigma_ {w} ^ {2} p _ {o}\right)}\right) (406) \\ \leq \exp \left(- \frac {t ^ {2}}{\Theta (r k \sigma_ {w} ^ {2})}\right), (407) \\ \end{array}
$$
where the inequality follows general Hoeffding's inequality.
For the second statement, when  $\mathbf{x} \sim \mathcal{X}_U$ , by Hoeffding's inequality,