SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \left| b _ {k} \right| \leq \frac {1}{\sqrt {\sum_ {j = 0} ^ {n} a _ {j} ^ {2}}} \sum_ {j = 0} ^ {n} \left| a _ {j} \right| \left| c _ {j, k} \right| \\ \leq \frac {1}{\sqrt {\sum_ {j = 0} ^ {n} a _ {j} ^ {2}}} \sum_ {j = 0} ^ {n} | a _ {j} | 2 ^ {\frac {j}{2}} \\ \leq \frac {1}{\sqrt {\sum_ {j = 0} ^ {n} a _ {j} ^ {2}}} \sqrt {\sum_ {j = 0} ^ {n} a _ {j} ^ {2}} \sqrt {\sum_ {j = 0} ^ {n} 2 ^ {j}} \\ <   \quad 2 ^ {\frac {n + 1}{2}} \\ \end{array}
$$
Now, let  $M_{j}$  be the maximal sum of coefficients of any polynomial computed by an output neuron of  $\Psi_{\vec{W}}^{j,n}$ . We next show by induction that  $M_{j} \leq (2\bar{d})^{2\sum_{k=1}^{j} n^{k}}$ . This will conclude the proof as it will imply that the sum of the coefficients of the polynomial computed by  $\Phi_{\vec{W}}^{i,n}$  is at most  $(2\bar{d})^{2} M_{i-1} \leq (2\bar{d})^{2\sum_{k=0}^{i-1} n^{k}} \leq (2\bar{d})^{4n^{i-1}}$ . For  $j = 0$  we have  $M_{0} = 1$ . For  $j \geq 1$  we have
$$
M _ {j} \leq \sum_ {k = 0} ^ {n} \left| b _ {k} \right| \left(\left(\bar {d}\right) ^ {2} M _ {j - 1}\right) ^ {k} \leq 2 ^ {\frac {n + 1}{2}} \cdot 2 \cdot \left(\left(\bar {d}\right) ^ {2} M _ {j - 1}\right) ^ {n} \leq \left(\left(2 \bar {d}\right) ^ {2} M _ {j - 1}\right) ^ {n}
$$
By the induction hypothesis we have
$$
M _ {j} \leq (2 \bar {d}) ^ {2 n + 2 n \sum_ {k = 1} ^ {j - 1} n ^ {k}} = (2 \bar {d}) ^ {2 \sum_ {k = 1} ^ {j} n ^ {k}}
$$
A.5 Proof of Lemma 2.4
$$
\begin{array}{l} \underset {W} {\mathbb {E}} \| f (W \mathbf {x}) - g (W \mathbf {y}) \| \overset {\text {J e s s e n I n e q u a l i t y}} {\leq} \sqrt {\underset {W} {\mathbb {E}} \| f (W \mathbf {x}) - g (W \mathbf {y}) \| ^ {2}} \\ = \sqrt {\frac {1}{d _ {2}} \sum_ {j = 1} ^ {d _ {2}} \mathbb {E} _ {W} \left(f ((W \mathbf {x}) _ {j}) - g ((W \mathbf {y}) _ {j}) ^ {2} \right.} \\ \end{array}
$$
Now, the lemma follows from the fact that  $\{((W\mathbf{x})_j,(W\mathbf{y})_j)\}_{j = 1}^{d_2}$  are independent centered Gaussian vectors with covariance matrix  $\begin{pmatrix} \| \mathbf{x}\| ^2 & \langle \mathbf{x},\mathbf{y}\rangle \\ \langle \mathbf{x},\mathbf{y}\rangle & \| \mathbf{y}\| ^2 \end{pmatrix}$ .
A.6 Proof of Lemma 2.8
Let  $P_A: \mathbb{R}^d \to A$  a function such that for any  $\mathbf{x} \in \mathbb{R}^d$ ,  $\| P_A(\mathbf{x}) - \mathbf{x} \| = d(\mathbf{x}, A)$ . Assume w.l.o.g. that  $\| \mathbf{x}_1 - P_A(\mathbf{x}_1) \| \leq \| \mathbf{x}_2 - P_A(\mathbf{x}_2) \|$ .
Case I:  $\| \mathbf{x}_2 - P_A(\mathbf{x}_2)\| \leq \| \mathbf{x}_1 - \mathbf{x}_2\|$
Simply define  $\tilde{\mathbf{x}}_i = P_A(\mathbf{x}_i)$ . We have
$$
\left\| \mathbf {x} _ {1} - \tilde {\mathbf {x}} _ {1} \right\| = \left\| \mathbf {x} _ {1} - P _ {A} (\mathbf {x} _ {1}) \right\|, \quad \left\| \mathbf {x} _ {2} - \tilde {\mathbf {x}} _ {2} \right\| = \left\| \mathbf {x} _ {2} - P _ {A} (\mathbf {x} _ {2}) \right\|
$$
$$
\left\| \tilde {\mathbf {x}} _ {1} - \tilde {\mathbf {x}} _ {2} \right\| \leq \left\| P _ {A} (\mathbf {x} _ {1}) - \mathbf {x} _ {1} \right\| + \left\| \mathbf {x} _ {1} - \mathbf {x} _ {2} \right\| + \left\| \mathbf {x} _ {2} - P _ {A} (\mathbf {x} _ {2}) \right\| \leq 3 \| \mathbf {x} _ {1} - \mathbf {x} _ {2} \|
$$
Case II:  $\| \mathbf{x}_1 - \mathbf{x}_2 \| \leq \| \mathbf{x}_2 - P_A(\mathbf{x}_2) \|$
Define  $\tilde{\mathbf{x}}_1 = \tilde{\mathbf{x}}_2 = P_A(\mathbf{x}_1)$ . We have
$$
\left\| \mathbf {x} _ {1} - \tilde {\mathbf {x}} _ {1} \right\| = \left\| \mathbf {x} _ {1} - P _ {A} (\mathbf {x} _ {1}) \right\|, \left\| \tilde {\mathbf {x}} _ {1} - \tilde {\mathbf {x}} _ {2} \right\| \leq 0 \| \mathbf {x} _ {1} - \mathbf {x} _ {2} \|
$$
$$
\left\| \mathbf {x} _ {2} - \tilde {\mathbf {x}} _ {2} \right\| \leq \left\| \mathbf {x} _ {2} - \mathbf {x} _ {1} \right\| + \left\| \mathbf {x} _ {1} - P _ {A} (\mathbf {x} _ {1}) \right\| \leq 2 \left\| \mathbf {x} _ {2} - P _ {A} (\mathbf {x} _ {2}) \right\|
$$
A.7 Proof of Lemma 2.12
Let  $B_{i,\delta}$  be the event that for some  $j < i$ ,  $|1 - \|\Psi_{\vec{W}}^{j}(\mathbf{x})\| | > \delta$  or  $|1 - \|\Psi_{\vec{W}}^{j,n}(\mathbf{x})\| | > \delta$ . We have
$$
\underset {\vec {W}} {\mathbb {E}} \left\| \Psi_ {\vec {W}} ^ {i} (\mathbf {x}) - \Psi_ {\vec {W}} ^ {i} (\mathbf {x}, \delta) \right\| = \underset {\vec {W}} {\mathbb {E}} \left[ \left\| \Psi_ {\vec {W}} ^ {i} (\mathbf {x}) \right\| 1 _ {B _ {i, \delta}} \right] \leq \sqrt {\underset {\vec {W}} {\mathbb {E}} \left[ \left\| \Psi_ {\vec {W}} ^ {i} (\mathbf {x}) \right\| ^ {2} \right]} \sqrt {\operatorname * {P r} (B _ {i , \delta})}
$$
$$
\frac {\mathbb {E}}{\vec {W}} \left\| \Psi_ {\vec {W}} ^ {i, n} (\mathbf {x}) - \Psi_ {\vec {W}} ^ {i, n} (\mathbf {x}, \delta) \right\| \leq \sqrt {\frac {\mathbb {E}}{\vec {W}} \left[ \left\| \Psi_ {\vec {W}} ^ {i , n} (\mathbf {x}) \right\| ^ {2} \right]} \sqrt {\operatorname * {P r} (B _ {i , \delta})}
$$
the lemma now follows from the following two claims.
Claim 1.  $\mathbb{E}_{\vec{W}}\left\| \Psi_{\vec{W}}^{i}(\mathbf{x})\right\|^{2}$  and  $\mathbb{E}_{\vec{W}}\left\| \Psi_{\vec{W}}^{i,n}(\mathbf{x})\right\|^{2}$  are bounded by a constant (independent of  $d_0,\ldots ,d_i)$ .
$$
\begin{array}{l} \underset {W ^ {i}} {\mathbb {E}} \left\| \Psi_ {\vec {W}} ^ {i} (\mathbf {x}) \right\| ^ {2} = \underset {\mathbf {w}} {\mathbb {E}} \sigma^ {2} \left(\mathbf {w} ^ {\top} \Psi_ {\vec {W}} ^ {i - 1} (\mathbf {x})\right) \\ \leq 2 \sigma^ {2} (0) + 2 L ^ {2} \underset {\mathbf {w}} {\mathbb {E}} \left(\mathbf {w} ^ {\top} \Psi_ {\vec {W}} ^ {i - 1} (\mathbf {x})\right) ^ {2} \\ = 2 \sigma^ {2} (0) + 2 L ^ {2} \| \Psi_ {\vec {W}} ^ {i - 1} (\mathbf {x}) \| ^ {2} \\ \end{array}
$$
By induction on  $i$ , this implies that  $\mathbb{E}_{\vec{W}}\left\| \Psi_{\vec{W}}^{i}(\mathbf{x})\right\|^{2}$  is bounded by a constant that depends only on  $i$  and  $L$  (but not on  $d_1,\ldots ,d_i$ ). For  $\mathbb{E}_{\vec{W}}\left\| \Psi_{\vec{W}}^{i,n}(\mathbf{x})\right\|^{2}$  we have
$$
\underset {W ^ {i}} {\mathbb {E}} \left\| \Psi_ {\vec {W}} ^ {i, n} (\mathbf {x}) \right\| ^ {2} = \underset {\mathbf {w}} {\mathbb {E}} \sigma_ {n} ^ {2} \left(\mathbf {w} ^ {\top} \Psi_ {\vec {W}} ^ {i - 1, n} (\mathbf {x})\right)
$$
Hence,  $\mathbb{E}_{W^i}\left\| \Psi_{\vec{W}}^{i,n}(\mathbf{x})\right\|^2$  is an even polynomial in  $\left\| \Psi_{\vec{W}}^{i - 1,n}(\mathbf{x})\right\|$  of degree  $\leq 2n$ . The polynomial depends only on  $\sigma_n$ . It therefore enough to show that for any  $i$  and  $k$ ,  $\mathbb{E}_{\vec{W}}\left\| \Psi_{\vec{W}}^{i,n}(\mathbf{x})\right\|^2 k$  is bounded,
by a bound that is independent of  $d_0, \ldots, d_i$ . We will show that via induction on  $i$ . For  $i = 0$  this is trivial as  $\left\| \Psi_{\vec{W}}^{0,n}(\mathbf{x}) \right\|^{2k} \equiv 1$ . Fix  $i \geq 1$ . We have
$$
\begin{array}{l} \underset {W ^ {i}} {\mathbb {E}} \left\| \Psi_ {\tilde {W}} ^ {i, n} (\mathbf {x}) \right\| ^ {2 k} = \underset {W ^ {i}} {\mathbb {E}} \left(\frac {\sum_ {j = 1} ^ {d _ {i}} \sigma_ {n} ^ {2} \left(\left(W ^ {i} \Psi_ {\tilde {W}} ^ {i - 1 , n} (\mathbf {x})\right) _ {j}\right)}{d _ {i}}\right) ^ {k} \\ \stackrel {\text {J e s s e n i n e q u a l i t y}} {\leq} \quad \frac {1}{d _ {i}} \underset {W ^ {i}} {\mathbb {E}} \sum_ {j = 1} ^ {d _ {i}} \sigma_ {n} ^ {2 k} \left(\left(W ^ {i} \Psi_ {\bar {W}} ^ {i - 1, n} (\mathbf {x})\right) _ {j}\right) \\ = \quad \mathop{\mathbb{E}}_{\mathbf{w}}\sigma_{n}^{2k}\left(\mathbf{w}^{\top}\Psi_{\vec{W}}^{i - 1,n}(\mathbf{x})\right) \\ \end{array}
$$
The last expression is an even polynomial in  $\| \Psi_{\vec{W}}^{i - 1,n}(\mathbf{x})\|$ . The polynomial depends only on  $2k$  and  $n$ . By the induction hypothesis we conclude that  $\mathbb{E}_{\vec{W}}\left\| \Psi_{\vec{W}}^{i,n}(\mathbf{x})\right\|^{2k}$  is bounded by a bound that is independent from  $d_0,\ldots ,d_i$ .
Claim 2. For every  $\delta, \epsilon'$ ,  $i$  and  $n$ , there is a constant  $D$  such that if  $d_1, \ldots, d_{i-1} \geq D$  then  $\operatorname{Pr}(B_{i,\delta}) < \epsilon'$ .
Proof. We will prove the lemma by induction on  $i$ . For  $i = 1$  this is immediate as  $\operatorname{Pr}(B_{i,\delta}) = 0$ . Fix  $i \geq 2$ . Let  $\delta'$  be small enough so that if  $||\mathbf{x}|| - 1| \leq \delta'$  then
$$
\left| \mathbb {E} _ {\mathbf {w}} \sigma^ {2} (\mathbf {w} ^ {\top} \mathbf {x}) - 1 \right| <   \frac {\delta}{4} \text {a n d} \left| \mathbb {E} _ {\mathbf {w}} \sigma_ {n} ^ {2} (\mathbf {w} ^ {\top} \mathbf {x}) - 1 \right| <   \frac {\delta}{4}
$$
$$
\left| \mathop {\mathbb {E}} _ {\mathbf {w}} \sigma^ {4} (\mathbf {w} ^ {\top} \mathbf {x}) - \mathop {\mathbb {E}} _ {X} \sigma^ {4} (X) \right| <   1 \text {a n d} \left| \mathop {\mathbb {E}} _ {\mathbf {w}} \sigma_ {n} ^ {4} (\mathbf {w} ^ {\top} \mathbf {x}) - \mathop {\mathbb {E}} _ {X} \sigma_ {n} ^ {4} (X) \right| <   1
$$
$$
\Pr \left(B _ {i, \delta}\right) \leq \Pr \left(B _ {i, \delta} \mid B _ {i - 1, \delta^ {\prime}} ^ {c}\right) + \Pr \left(B _ {i - 1, \delta^ {\prime}}\right)
$$
By Chebyshev inequality,  $\operatorname{Pr}(B_{i,\delta}|B_{i - 1,\delta^{\prime}}^{c}) < \frac{\epsilon^{\prime}}{2}$  for sufficiently large  $d_{i - 1}$ . By the induction hypothesis,  $\operatorname{Pr}(B_{i - 1,\delta^{\prime}}) < \frac{\epsilon^{\prime}}{2}$  for sufficiently large  $d_{1},\ldots ,d_{i - 2}$