SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathbb {E} _ {x} \left[ \left(f (x; \hat {\theta}) - f ^ {*} (x)\right) ^ {2} \right] - \mathbb {E} _ {x} \left[ \ell_ {\tau} \left(f (x; \hat {\theta}) - f ^ {*} (x)\right) \right] \\ = \mathbb {E} _ {x} \left[ \left(\left(f (x; \hat {\theta}) - f ^ {*} (x)\right) ^ {2} - \tau^ {2}\right) \cdot \mathbf {1} _ {| f (x; \hat {\theta}) - f ^ {*} (x) | \geq \tau} \right] \\ \leq \mathbb {E} _ {x} \left[ \left(f (x; \hat {\theta}) - f ^ {*} (x)\right) ^ {2} \cdot \mathbf {1} _ {\left| f (x; \hat {\theta}) - f ^ {*} (x) \right| \geq \tau} \right] \\ \leq \mathbb {E} _ {x} \left[ \left(f (x; \hat {\theta}) - f ^ {*} (x)\right) ^ {4} \right] ^ {1 / 2} \cdot \mathbb {P} \left(\left| f (x; \hat {\theta}) - f ^ {*} (x) \right| \geq \tau\right) ^ {1 / 2} \\ \lesssim \left[ \mathbb {E} _ {x} \left[ f (x; \hat {\theta}) ^ {4} \right] ^ {1 / 2} + \mathbb {E} _ {x} \left[ f ^ {*} (x) ^ {4} \right] ^ {1 / 2} \right] \cdot \left[ \mathbb {P} \Big (\big | f (x; \hat {\theta}) \big | \geq \tau \Big) + \mathbb {P} (| f ^ {*} (x) | \geq \tau) \right] \\ \end{array}
$$
Next, we bound  $f(x; \hat{\theta})$ :
$$
\begin{array}{l} \left| f (x; \hat {\theta}) \right| = \left| \frac {1}{m _ {1}} \sum_ {j = 1} ^ {m _ {1}} \hat {a} _ {j} \sigma_ {1} \left(a _ {j} ^ {(0)} \cdot \bar {\eta} \phi (x) + b _ {j}\right) \mathbf {1} _ {b _ {j} ^ {(0)} > 0} \right| \\ \leq \frac {1}{m _ {1}} \sum_ {j = 1} ^ {m _ {1}} | \hat {a} _ {j} | (| \bar {\eta} \phi (x) | + | b _ {j} |) \\ \leq \frac {\| \hat {a} \| _ {2}}{\sqrt {m _ {1}}} | \bar {\eta} \phi (x) | + \frac {1}{m _ {1}} \| \hat {a} \| _ {2} \| b \| _ {2} \\ \end{array}
$$
By Lemma 10, with high probability over the initialization we have  $\| b\| _2\lesssim \sqrt{m_1}$ . Next, by Lemma 4, with high probability over the initialization and dataset we have  $\| \hat{a}\| _2\lesssim \| a^*\| _2\lesssim \| q\|_{2,\infty}\sqrt{m_1}$ . Thus with high probability we have
$$
\left| f (x; \hat {\theta}) \right| \lesssim \| q \| _ {2, \infty} (| \bar {\eta} \phi (x) | + 1)
$$
uniformly over  $x$ .
We naively bound
$$
\left| \phi (x) \right| \leq \frac {1}{m _ {2}} \left\| h ^ {(0)} (x) \right\| \cdot \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h ^ {(0)} \left(x _ {i}\right) \right\| \left| f \left(x _ {i}; \theta^ {(0)}\right) - f ^ {*} \left(x _ {i}\right) \right|.
$$
By Lemma 11 and Lemma 12, with high probability we have  $\left|f(x_{i};\theta^{(0)}) - f^{*}(x_{i})\right|\lesssim \iota^{\ell}$  for all  $i$ . With high probability, we also have  $\| h^{(0)}(x_i)\| \lesssim \sqrt{m_2}$ . Additionally, we have
$$
\begin{array}{l} \left\| h ^ {(0)} (x) \right\| _ {2} ^ {4} = \left(\sum_ {j = 1} ^ {m _ {2}} \sigma_ {2} (x \cdot v _ {j}) ^ {2}\right) ^ {2} \\ \leq m _ {2} \sum_ {j = 1} ^ {m _ {2}} \sigma_ {2} (x \cdot v _ {j}) ^ {4} \\ \lesssim m _ {2} \sum_ {j = 1} ^ {m _ {2}} \left(1 + \left| x \cdot v _ {j} \right| ^ {4 \alpha_ {\sigma}}\right) \\ \end{array}
$$
Since  $x$  is  $C_{\gamma}$  subGaussian and  $\| v \| = 1$ , we have
$$
\begin{array}{l} \mathbb {E} _ {x} \left\| h ^ {(0)} (x) \right\| _ {2} ^ {4} \lesssim m _ {2} ^ {2} \mathbb {E} | x \cdot v _ {j} | ^ {4 \alpha \sigma} \\ \lesssim m _ {2} ^ {2}. \\ \end{array}
$$
$$
\mathbb {E} _ {x} | \phi (x) | ^ {4} \lesssim \iota^ {4 \ell},
$$
$$
\mathbb {E} _ {x} \left[ \left| f (x; \hat {\theta}) \right| ^ {4} \right] ^ {1 / 2} \lesssim \| q \| _ {2, \infty} ^ {2} \overline {{\eta}} ^ {2} \iota^ {2 \ell} \leq \| q \| _ {2, \infty} ^ {2} m _ {1} ^ {2} \| \mathbb {K} f ^ {*} \| _ {L ^ {2}} ^ {- 2} \iota^ {2 \ell}
$$
By Assumption 2,  $\mathbb{E}_x\left[(f^* (x))^4\right]\lesssim 1$
We choose  $\tau = \Theta (\max (\iota^{\ell},\| q\|_{2,\infty})$ . By Lemma 7 and Lemma 8  $\left\{\left|f(x,\hat{\theta})\right| > \tau \right\}$  and  $\{|f^{*}(x)| > \tau \}$  are both high probability events. Therefore by choosing  $C$  sufficiently large, we have the bound
$$
\left[ \mathbb {P} \left(\left| f (x; \hat {\theta}) \right| \geq \tau\right) + \mathbb {P} \left(| f ^ {*} (x) | \geq \tau\right) \right] \leq m _ {1} ^ {- 4} \iota^ {- 2 \ell}
$$
Altogether, this gives
$$
\mathbb {E} _ {x} \left[ \left(f (x; \hat {\theta}) - f ^ {*} (x)\right) ^ {2} \right] - \mathbb {E} _ {x} \left[ \ell_ {\tau} \Big (f (x; \hat {\theta}) - f ^ {*} (x) \Big) \right] \lesssim \frac {\| q \| _ {2 , \infty} ^ {2} \| \mathbb {K} f ^ {*} \| _ {L ^ {2}} ^ {- 2}}{m _ {1} ^ {2}} + \frac {1}{m _ {1} ^ {4}} \leq \frac {\| q \| _ {2 , \infty} ^ {2}}{m _ {1}}.
$$
D.4 Auxiliary Lemmas
Proof of Lemma 7. We can decompose
$$
\begin{array}{l} | \phi (x) - (\mathbb {K} f ^ {*}) (x) | = \left| \frac {1}{m _ {2}} \cdot \frac {1}{n} \sum_ {i = 1} ^ {n} \left(f \left(x _ {i}; \theta^ {(0)}\right) - f ^ {*} \left(x _ {i}\right)\right) \langle h ^ {(0)} (x _ {i}), h ^ {(0)} (x) \rangle - (\mathbb {K} f ^ {*}) (x) \right| \\ \leq \frac {1}{m _ {2} n} \sum_ {i = 1} ^ {n} \left| f (x _ {i}; \theta^ {(0)}) \langle h ^ {(0)} (x _ {i}), h ^ {(0)} (x) \rangle \right| \\ + \frac {1}{m _ {2}} \left| \frac {1}{n} \sum_ {i = 1} ^ {n} f ^ {*} (x _ {i}) \langle h ^ {(0)} (x _ {i}), h ^ {(0)} (x) \rangle - \left\langle \mathbb {E} _ {x} \left[ f ^ {*} (x) h ^ {(0)} (x) \right], h ^ {(0)} (x) \right\rangle \right| \\ + \left| \frac {1}{m _ {2}} \left\langle \mathbb {E} _ {x} \left[ f ^ {*} (x) h ^ {(0)} (x) \right], h ^ {(0)} (x) \right\rangle - (\mathbb {K} f ^ {*}) (x) \right|. \\ \end{array}
$$
By Lemma 11 and Lemma 13 we can upper bound the first term by  $\sqrt{\frac{\iota}{m_1}}$ .
For the second term, by Lemma 13 and Lemma 15 we have for all  $x \in \mathcal{D}_2$
$$
\frac {1}{m _ {2}} \left\| \frac {1}{n} \sum_ {i = 1} ^ {n} f ^ {*} (x _ {i}) h ^ {(0)} (x _ {i}) - \mathbb {E} _ {x} \Big [ f ^ {*} (x) h ^ {(0)} (x) \Big ] \right\| _ {2} \left\| h ^ {(0)} (x) \right\| _ {2} \lesssim \frac {1}{m _ {2}} \sqrt {m _ {2}} \cdot \sqrt {\frac {m _ {2} \iota^ {2 \ell + 2 \alpha_ {\sigma} + 1}}{n}} \lesssim \sqrt {\frac {\iota^ {2 \ell + 2 \alpha_ {\sigma} + 1}}{n}}.
$$
The third term can be bounded as
$$
\left| \frac {1}{m _ {2}} \sum_ {i = 1} ^ {m _ {2}} \mathbb {E} _ {x ^ {\prime}} [ f ^ {*} (x ^ {\prime}) \sigma_ {2} (x ^ {\prime} \cdot v _ {j}) \sigma_ {2} (x \cdot v _ {j}) ] - \mathbb {E} _ {x ^ {\prime}, v} [ f ^ {*} (x) \sigma_ {2} (x ^ {\prime} \cdot v) \sigma_ {2} (x \cdot v) ] \right| \lesssim \sqrt {\frac {\iota^ {2 \alpha_ {\sigma} + 1}}{m _ {2}}}
$$
Proof of Lemma 8. Conditioning on the event where Corollary 1 holds, and with the choice  $\overline{\eta} = \frac{1}{2} C_K^{-1}e^{-1}\| \mathbb{K}f^*\|_{L^2}^{-1}\iota^{-\chi}$ , we get that with high probability
$$
\sup  _ {x \in \mathcal {D} _ {2}} | (\bar {\eta} \cdot \mathbb {K} f ^ {*}) (x) | \leq \frac {1}{2}.
$$
$$
\begin{array}{l} \sup  _ {x \in \mathcal {D} _ {2}} | \bar {\eta} \phi (x) | = C _ {K} ^ {- 1} e ^ {- 1} \| \mathbb {K} f ^ {*} \| _ {L ^ {2}} ^ {- 1} \iota^ {- \chi} \cdot \sup  _ {x} | \phi (x) | \\ \leq C _ {K} ^ {- 1} e ^ {- 1} \| \mathbb {K} f ^ {*} \| _ {L ^ {2}} ^ {- 1} \iota^ {- \chi} \left[ \sup  _ {x} | (\mathbb {K} f ^ {*}) (x) | + O \left(\sqrt {\frac {\iota^ {2 \ell + 2 \alpha_ {\sigma} + 1}}{n}} + \sqrt {\frac {\iota^ {2 \alpha_ {\sigma} + 1}}{m _ {2}}} + \sqrt {\frac {\iota}{m _ {1}}}\right) \right] \\ \leq \frac {1}{2} + O \left(\| \mathbb {K} f ^ {*} \| _ {L ^ {2}} ^ {- 1} \iota^ {- \chi} \cdot \left(\sqrt {\frac {\iota^ {2 \ell + 2 \alpha_ {\sigma} + 1}}{n}} + \sqrt {\frac {\iota^ {2 \alpha_ {\sigma} + 1}}{m _ {2}}} + \sqrt {\frac {\iota}{m _ {1}}}\right)\right) \\ \leq 1. \\ \end{array}
$$
Proof of Lemma 9. Conditioning on the event where Lemma 8 holds and applying Lemma 3, we get that there exists  $v$  such that
$$
\mathbb {E} _ {a, b} [ v (a, b) \sigma_ {1} (a \bar {\eta} \phi (x) + b) ] = q (\bar {\eta} \phi (x))
$$
for all  $x\in \mathcal{D}_2$  . Since  $v(a,b) = 0$  for  $b\leq 0$  , we have that  $v(a,b) = v(a,b)\mathbf{1}_{b > 0}$  . Thus the desired claim is true.
D.5 Concentration
Lemma 10. Let  $m_1 \gtrsim \iota$ . With high probability,
$$
\frac {1}{m _ {1}} \sum_ {i = 1} ^ {m _ {1}} \left(b _ {i} ^ {(0)}\right) ^ {2} \lesssim 1.
$$
Proof. By Bernstein, we have
$$
\left| \frac {1}{m _ {1}} \sum_ {i = 1} ^ {m _ {1}} \left(b _ {i} ^ {(0)}\right) ^ {2} - 1 \right| \lesssim \sqrt {\frac {\iota}{m _ {1}}} \leq 1.
$$
Lemma 11. With high probability,  $|f(x; \theta^{(0)})| \lesssim \sqrt{\frac{\iota}{m_1}}$ . for all  $x$ .
$$
f (x; \theta^ {(0)}) = \frac {1}{m _ {1}} \sum_ {i = 1} ^ {m _ {1}} a _ {i} \sigma_ {1} (b _ {i}).
$$
Since  $a_i \sim \mathrm{Unif}(\{\pm 1\})$  and  $b_i \sim \mathcal{N}(0,1)$ , the quantities  $a_i \sigma_1(b_i)$  are 1-subGaussian, and thus by Hoeffding with high probability we have
$$
\left| f (x; \theta^ {(0)}) \right| \lesssim \sqrt {\frac {\iota}{m _ {1}}}.
$$
Lemma 12. With high probability
$$
\sup  _ {x \in \mathcal {D} _ {2}} \mathbb {E} _ {v} \left[ \sigma_ {2} (x \cdot v) ^ {4} \right] \lesssim 1 \quad a n d \quad \sup  _ {j \in [ m _ {2} ]} \mathbb {E} _ {x} \left[ \sigma_ {2} (x \cdot v _ {j}) ^ {4} \right] \lesssim 1.
$$
Proof. First, we have that  $\sigma_2(x\cdot v)^4\lesssim 1 + (x\cdot v)^{4\alpha_\sigma}$ . Therefore
$$
\mathbb {E} _ {v} \big [ \sigma_ {2} (x \cdot v) ^ {4} \big ] \lesssim 1 + \mathbb {E} _ {v} \big [ (x \cdot v) ^ {4 \alpha_ {\sigma}} \big ] \lesssim 1 + \| x \| ^ {4 \alpha_ {\sigma}} d ^ {- 2 \alpha_ {\sigma}}.
$$
Next, with high probability we have  $\sup_{x\in \mathcal{D}_2}\left|\| x\|^2 -\mathbb{E}\| x\|^2\right|\leq \iota C_\gamma^2\lesssim \iota$  . Since  $\mathbb{E}\| x\| ^2\lesssim d\gamma_x^2\lesssim d$  we can thus bound
$$
\begin{array}{l} \sup  _ {x \in \mathcal {D} _ {2}} \mathbb {E} _ {v} \left[ \sigma_ {2} (x \cdot v) ^ {4} \right] \lesssim 1 + \gamma_ {v} ^ {4 \alpha_ {\sigma}} \sup  _ {x \in \mathcal {D} _ {2}} \| x \| ^ {2} \\ \lesssim 1 + d ^ {- 2 \alpha_ {\sigma}} (d + \iota) ^ {2 \alpha_ {\sigma}} \\ \lesssim 1. \\ \end{array}
$$
The proof for the other inequality is identical.
Lemma 13. Let  $m_2 \gtrsim \iota^{2\alpha_\sigma + 1}$ . With high probability,  $\sup_{x \in \mathcal{D}_1 \cup \mathcal{D}_2} \| h^{(0)}(x) \| \lesssim \sqrt{m_2}$
Proof. Since  $x$  is  $C_{\gamma}$  subGaussian and  $v$  is  $1 / \sqrt{d}$  subGaussian, with high probability we have
$$
x \cdot v \leq O (C _ {\gamma} \iota) = O (\iota).
$$
Union bounding over  $x \in \mathcal{D}_2, j \in [m_2]$ , with high probability we have  $\sup_{x \in \mathcal{D}_1 \cup \mathcal{D}_2, j \in [m_2]} |x \cdot v_j| \leq O(\iota)$ . Therefore
$$
\sup  _ {x, v _ {j}} | \sigma_ {2} (x \cdot v _ {j}) | \leq O (\iota^ {\alpha_ {\sigma}}).
$$
Next, see that
$$
\frac {1}{m _ {2}} \left\| h ^ {(0)} (x) \right\| ^ {2} = \frac {1}{m _ {2}} \sum_ {j = 1} ^ {m _ {2}} \sigma_ {2} (x \cdot v _ {j}) ^ {2}.
$$
Pick truncation radius  $R = O(\iota^{\alpha_{\sigma}})$ . By Hoeffding's inequality and a union bound over  $x \in \mathcal{D}_1 \cup \mathcal{D}_2$ , we have that with high probability
$$
\sup _ {x \in \mathcal {D} _ {1} \cup \mathcal {D} _ {2}} \left| \frac {1}{m _ {2}} \sum_ {j = 1} ^ {m _ {2}} \mathbf {1} _ {\sigma_ {2} (x \cdot v _ {j}) ^ {2} \leq R} \sigma_ {2} (x \cdot v _ {j}) ^ {2} - \mathbb {E} _ {v} \big [ \mathbf {1} _ {\sigma_ {2} (x \cdot v _ {j}) ^ {2} \leq R} \sigma_ {2} (x \cdot v) ^ {2} \big ] \right| \lesssim \sqrt {\frac {\iota^ {2 \alpha_ {\sigma} + 1}}{m _ {2}}}.
$$
$$
\begin{array}{l} \left| \mathbb {E} _ {v} \left[ \mathbf {1} _ {\sigma_ {2} (x \cdot v) ^ {2} \leq R} \sigma_ {2} (x \cdot v) ^ {2} \right] - \mathbb {E} _ {v} \left[ \sigma (x \cdot v) ^ {2} \right] \right| = \mathbb {E} _ {v} \left[ \mathbf {1} _ {\sigma_ {2} (x \cdot v) ^ {2} > R} \sigma_ {2} (x \cdot v) ^ {2} \right] \\ \leq \mathbb {P} \left(\sigma_ {2} (x \cdot v) ^ {2} > R\right) \mathbb {E} \left[ \sigma_ {2} (x \cdot v) ^ {4} \right] ^ {1 / 2} \\ \leq \frac {1}{d}. \\ \end{array}
$$
Therefore with high probability
$$
\left| \frac {1}{m _ {2}} \left\| h ^ {(0)} (x) \right\| ^ {2} - \mathbb {E} _ {v} \left[ \sigma (x \cdot v) ^ {2} \right] \right| \lesssim \sqrt {\frac {\iota^ {2 \alpha_ {\sigma} + 1}}{m _ {2}}} + \frac {1}{d}.
$$
by Lemma 12. Lemma 12 also tells us that  $\mathbb{E}_v[\sigma_2(x\cdot v)^2 ] = O(1)$ . Altogether, for  $m_{2}\gtrsim \iota^{2\alpha_{\sigma} + 1}$  we have
$$
\frac {1}{m _ {2}} \left\| h ^ {(0)} (x) \right\| ^ {2} \leq 2
$$
and hence  $\| h^{(0)}(x)\| \lesssim \sqrt{m_2}$ , as desired.
Lemma 14.  $\mathbb{P}_{x\sim \nu}\big(|f^{*}(x)|\geq C_{f}e\iota^{\ell}\big)\leq e^{-\iota}$  and  $\mathbb{P}_{x\sim \nu}(|(\mathbb{K}f^{*})(x)|\geq C_{K}e\iota^{\chi}\cdot \| \mathbb{K}f^{*}\|_{L^{2}})\leq e^{-\iota}$
Proof. By Markov's inequality, we have
$$
\mathbb {P} (| f ^ {*} (x) | > t) = \mathbb {P} (| f ^ {*} (x) | ^ {q} > t ^ {q}) \leq \frac {\| f ^ {*} \| _ {q} ^ {q}}{t ^ {q}} \leq \frac {C _ {f} ^ {q} q ^ {q \ell}}{t ^ {q}}.
$$
Choose  $t = C_f e^{\iota \ell}$ . We select  $q = \frac{\iota}{e^{1 - 1 / \ell}}$ , which is at least 1 for  $C$  in the definition of  $\iota$  sufficiently large. Plugging in, we get
$$
\mathbb {P} \big (| f ^ {*} (x) | > C _ {f} e \iota^ {\ell} \big) \leq \frac {C _ {f} ^ {q} \iota^ {q \ell}}{C _ {f} ^ {q} e ^ {q \ell} \iota^ {q \ell}} = e ^ {- q \ell} = e ^ {- \iota \ell e ^ {1 / \ell - 1}} \leq e ^ {- \iota},
$$
since  $\ell e^{1 / \ell -1}\geq 1$
An analogous derivation for the function  $\frac{\mathbb{K}f^*}{\|\mathbb{K}f^*\|_{L^2}}$  yields the second bound
Corollary 1. With high probability,  $\sup_{x\in \mathcal{D}_2}|f^* (x)|\leq C_fe\cdot \iota^\ell$  and  $\sup_{x\in \mathcal{D}_2}(|(\mathbb{K}f^*)(x)|\lesssim C_Ke\cdot$ $\iota^{\chi}\| \mathbb{K}f^{*}\|_{L^{2}}$
Proof. Union bounding the previous lemma over  $x \in \mathcal{D}_2$  yields the desired result.
Lemma 15. With high probability,
$$
\left\| \frac {1}{n} \sum_ {i = 1} ^ {n} f ^ {*} (x _ {i}) h ^ {(0)} (x _ {i}) - \mathbb {E} _ {x} \Big [ f ^ {*} (x) h ^ {(0)} (x) \Big ] \right\| _ {2} \lesssim \sqrt {\frac {m _ {2} \ell^ {2 \ell + 2 \alpha_ {\sigma} + 1}}{n}}
$$
Proof. Consider the quantity
$$
\left| \frac {1}{n} \sum_ {i = 1} ^ {n} f ^ {*} (x _ {i}) \sigma_ {2} (x _ {i} \cdot v _ {j}) - \mathbb {E} _ {x} \left[ f ^ {*} (x) h ^ {(0)} (x _ {i}) \right] \right|.
$$
With high probability,  $\sup_{x\in \mathcal{D}_2,j\in [m_2]}|f^* (x_i)\sigma_2(x_i\cdot v_j)|\lesssim \iota^{\ell +\alpha_\sigma}$ . Pick truncation radius  $R = O(\iota^{\ell +\alpha_{\sigma}})$ . By Hoeffding, we have with high probability.
$$
\left| \frac {1}{n} \sum_ {i = 1} ^ {n} f ^ {*} (x _ {i}) \sigma_ {2} (x _ {i} \cdot v _ {j}) {\bf 1} _ {f ^ {*} (x _ {i}) \sigma_ {2} (x _ {i} \cdot v _ {j}) \leq R} - \mathbb {E} _ {x} \big [ f ^ {*} (x) \sigma_ {2} (x _ {i} \cdot v _ {j}) {\bf 1} _ {f ^ {*} (x _ {i}) \sigma_ {2} (x _ {i} \cdot v _ {j}) \leq R} \big ] \right| \lesssim \sqrt {\frac {\iota^ {2 \ell + 2 \alpha_ {\sigma} + 1}}{n}}.
$$
$$
\begin{array}{l} \left| \mathbb {E} _ {x} \left[ f ^ {*} (x) \sigma_ {2} (x \cdot v _ {j}) \mathbf {1} _ {f ^ {*} (x _ {i}) \sigma_ {2} (x \cdot v _ {j}) \leq R} \right] - \mathbb {E} _ {x} \left[ f ^ {*} (x) \sigma_ {2} (x _ {i} \cdot v _ {j}) \right] \right| \\ = \left| \mathbb {E} \left[ \mathbf {1} _ {f ^ {*} (x) \sigma_ {2} (x \cdot v _ {j}) > R} f ^ {*} (x) \sigma_ {2} \left(x _ {i} \cdot v _ {j}\right) \right] \right| \\ \leq \mathbb {P} (f ^ {*} (x) \sigma_ {2} (x \cdot v _ {j}) > R) \mathbb {E} _ {x} \left[ \sigma_ {2} (x \cdot v _ {j}) ^ {4} \right] \\ \lesssim \frac {1}{n}. \\ \end{array}
$$
$$
\left| \frac {1}{n} \sum_ {i = 1} ^ {n} f ^ {*} (x _ {i}) \sigma_ {2} (x _ {i} \cdot v _ {j}) - \mathbb {E} _ {x} \left[ f ^ {*} (x) h ^ {(0)} (x _ {i}) \right] \right| \lesssim \sqrt {\frac {\iota^ {2 \ell + 2 \alpha_ {\sigma} + 1}}{n}}
$$
By a union bound, the above holds with high probability for all  $j \in [m_2]$ , and thus
$$
\left\| \frac {1}{n} \sum_ {i = 1} ^ {n} f ^ {*} (x _ {i}) h ^ {(0)} (x _ {i}) - \mathbb {E} _ {x} \left[ f ^ {*} (x) h ^ {(0)} (x) \right] \right\| _ {2} \lesssim \sqrt {\frac {m _ {2} t ^ {2 \ell + 2 \alpha_ {\sigma} + 1}}{n}}
$$
Lemma 16. With high probability,
$$
\sup _ {x \in \mathcal {X} _ {d}} \left| \frac {1}{m _ {2}} \sum_ {i = 1} ^ {m _ {2}} \mathbb {E} _ {x ^ {\prime}} [ f ^ {*} (x ^ {\prime}) \sigma_ {2} (x ^ {\prime} \cdot v _ {j}) \sigma_ {2} (x \cdot v _ {j}) ] - \mathbb {E} _ {x ^ {\prime}, v} [ f ^ {*} (x) \sigma_ {2} (x ^ {\prime} \cdot v) \sigma_ {2} (x \cdot v) ] \right| \lesssim \sqrt {\frac {l ^ {2 \alpha_ {\sigma} + 1}}{m _ {2}}}
$$
Proof. Fix  $x \in \mathcal{D}_2$ . Consider the random variables  $Z(v) = \mathbb{E}_{x'}[f^*(x')\sigma_2(x' \cdot v)\sigma_2(x \cdot v)]$ . With high probability we have  $|\sigma_2(x \cdot v_j)| \lesssim \iota^{\alpha_\sigma}$  and thus
$$
| Z (v _ {j}) | = | \mathbb {E} _ {x ^ {\prime}} [ f ^ {*} (x ^ {\prime}) \sigma_ {2} (x ^ {\prime} \cdot v _ {j}) \sigma_ {2} (x \cdot v _ {j}) ] | \lesssim \iota^ {\alpha_ {\sigma}} \cdot \mathbb {E} _ {x ^ {\prime}} \left[ f ^ {*} (x ^ {\prime}) ^ {2} \right] ^ {1 / 2} \mathbb {E} \left[ \sigma_ {2} (x ^ {\prime} \cdot v _ {j}) ^ {2} \right] ^ {2} \lesssim \iota^ {\alpha_ {\sigma}}.
$$
For all  $j \in [m_2]$ . Choosing truncation radius  $R = \iota^{\alpha_{\sigma}}$ , with Hoeffding we have with high probability that
$$
\left| \frac {1}{m _ {2}} \sum_ {i = 1} ^ {m _ {2}} \mathbb {E} _ {x ^ {\prime}} [ f ^ {*} (x ^ {\prime}) \sigma_ {2} (x ^ {\prime} \cdot v _ {j}) ] \sigma_ {2} (x \cdot v _ {j}) \mathbf {1} _ {| Z (v _ {j}) | \leq R} - \mathbb {E} _ {x ^ {\prime}, v} [ f ^ {*} (x ^ {\prime}) \sigma_ {2} (x ^ {\prime} \cdot v) \sigma_ {2} (x \cdot v) \mathbf {1} _ {| Z (v) | \leq R} ] \right| \lesssim \sqrt {\frac {t ^ {2 \alpha_ {\sigma} + 1}}{m _ {2}}}
$$
Next, we have that
$$
\begin{array}{l} \left. \right.\left| \mathbb {E} _ {x ^ {\prime}, v} \left[ f ^ {*} \left(x ^ {\prime}\right) \sigma_ {2} \left(x ^ {\prime} \cdot v\right) \sigma_ {2} (x \cdot v) \mathbf {1} _ {| Z (v) | \leq R} \right] - \mathbb {E} _ {x ^ {\prime}, v} \left[ f ^ {*} \left(x ^ {\prime}\right) \sigma_ {2} \left(x ^ {\prime} \cdot v\right) \sigma_ {2} (x \cdot v) \right]\right| \\ = \left| \mathbb {E} _ {x ^ {\prime}, v} \left[ f ^ {*} \left(x ^ {\prime}\right) \sigma_ {2} \left(x ^ {\prime} \cdot v\right) \sigma_ {2} (x \cdot v) \mathbf {1} _ {| Z (v) | > R} \right] \right| \\ \leq \mathbb {P} (Z (v) > R) \cdot \mathbb {E} _ {x ^ {\prime}, v} \left[ f ^ {*} \left(x ^ {\prime}\right) ^ {2} \sigma_ {2} \left(x ^ {\prime} \cdot v\right) ^ {2} \sigma_ {2} \left(x \cdot v\right) ^ {2} \right] ^ {1 / 2} \\ \lesssim \frac {1}{m _ {2}}. \\ \end{array}
$$
Conditioning on the high probability event that  $\sup_{j\in [m_2]}|Z(v_j)|\leq R$ , we have with high probability that
$$
\left| \frac {1}{m _ {2}} \sum_ {i = 1} ^ {m _ {2}} \mathbb {E} _ {x ^ {\prime}} [ f ^ {*} (x ^ {\prime}) \sigma_ {2} (x ^ {\prime} \cdot v _ {j}) ] \sigma_ {2} (x \cdot v _ {j}) - \mathbb {E} _ {x ^ {\prime}, v} [ f ^ {*} (x ^ {\prime}) \sigma_ {2} (x ^ {\prime} \cdot v) \sigma_ {2} (x \cdot v) ] \right| \lesssim \sqrt {\frac {\ell^ {2 \alpha_ {\sigma} + 1}}{m _ {2}}}.
$$
A union bound over  $x \in \mathcal{D}_2$  yields the desired result.
Lemma 17. With high probability,
$$
\sup _ {x \in \mathcal {D} _ {2}} \left| \frac {1}{m _ {1}} \sum_ {i = 1} ^ {m _ {1}} v (a _ {i}, b _ {i}) \sigma_ {1} (a _ {i} \overline {{\eta}} \phi (x) + b _ {i}) {\bf 1} _ {b _ {i} > 0} - f _ {v} ^ {\infty} (x) \right| \lesssim \sqrt {\frac {\| v \| _ {\infty} ^ {2} \mu}{m _ {1}}}
$$
Proof. Condition on the high probability event  $\sup_{x\in \mathcal{D}_2}|\overline{\eta}\phi (x)|\leq 1$ . Next, note that whenever  $b > 2$  that  $v(a,b) = 0$ . Therefore we can bound
$$
\left| v (a, b) \sigma_ {1} \left(a \bar {\eta} \phi (x) + b _ {i}\right) \mathbf {1} _ {b _ {i} > 0} \right| \leq 2 \| v \| _ {\infty}
$$
Therefore by Hoeffding's inequality we have that
$$
\left| \frac {1}{m _ {1}} \sum_ {i = 1} ^ {m _ {1}} v (a _ {i}, b _ {i}) \sigma_ {1} (a _ {i} \overline {{\eta}} \phi (x) + b _ {i}) {\bf 1} _ {b _ {i} > 0} - f _ {v} ^ {\infty} (x) \right| \lesssim \sqrt {\frac {\| v \| _ {\infty} ^ {2} \iota}{m _ {1}}} +
$$
The desired result follows via a Union bound over  $x \in \mathcal{D}_2$ .
Lemma 18. With high probability,
$$
\left| \frac {1}{m _ {1}} \sum_ {i = 1} ^ {m _ {1}} v (a _ {i}, b _ {i}) ^ {2} - \| v \| _ {L ^ {2}} ^ {2} \right| \lesssim \sqrt {\frac {\| v \| _ {\infty} ^ {4} \iota}{m _ {1}}}.
$$
$$
v (a _ {i}, b _ {i}) ^ {2} \leq \| v \| _ {\infty} ^ {2}
$$
Thus by Hoeffding's inequality we have that
Corollary 2. Let  $m_1 \gtrsim \iota$ . Then with high probability
$$
\sum_ {i = 1} ^ {m _ {i}} v \left(a _ {i}, b _ {i}\right) ^ {2} \lesssim \| v \| _ {\infty} ^ {2} m _ {1}.
$$
Proof. By the previous lemma, we have that
$$
\left| \frac {1}{m _ {1}} \sum_ {i = 1} ^ {m _ {1}} v (a _ {i}, b _ {i}) ^ {2} - \| v \| _ {L ^ {2}} ^ {2} \right| \lesssim \sqrt {\frac {\| v \| _ {\infty} ^ {4} \iota}{m _ {1}}} \lesssim \| v \| _ {\infty} ^ {2}.
$$
$$
\sum_ {i = 1} ^ {m _ {i}} v (a _ {i}, b _ {i}) ^ {2} \lesssim \| v \| _ {L ^ {2}} ^ {2} m _ {1} + \| v \| _ {\infty} ^ {2} m _ {1} \lesssim \| v \| _ {\infty} ^ {2} m _ {1}.
$$
Lemma 19. With high probability,
Proof. Let  $S$  be the set of  $x$  so that  $|\overline{\eta}(\mathbb{K}f^*)(x)| \leq 1$  and  $|f^*(x)| \lesssim \iota^\ell$ . Consider the random variables  $(q(\overline{\eta}(\mathbb{K}f^*)(x)) - f^*(x))^2 \cdot \mathbf{1}_{x \in S}$ . We have that
$$
\left| \left(q (\overline {{\eta}} (\mathbb {K} f ^ {*}) (x)) - f ^ {*} (x)\right) ^ {2} \cdot \mathbf {1} _ {x \in S} \right| \lesssim \sup  _ {z \in [ - 1, 1 ]} | q (z) | ^ {2} + \iota^ {2 \ell}.
$$
$$
\mathbb {E} \left[ \left(q (\bar {\eta} (\mathbb {K} f ^ {*}) (x)) - f ^ {*} (x)\right) ^ {2} \cdot \mathbf {1} _ {x \in S} \right] ^ {2} \lesssim \left(\sup  _ {z \in [ - 1, 1 ]} | q (z) | ^ {2} + \iota^ {2 \ell}\right) \cdot \| q \circ (\bar {\eta} (\mathbb {K} f ^ {*})) - f ^ {*} \| _ {L ^ {2}} ^ {2}.
$$
Therefore by Bernstein's inequality we have that
$$
\begin{array}{l} \left| \frac {1}{n} \sum_ {x \in \mathcal {D} _ {2}} \left(q (\bar {\eta} (\mathbb {K} f ^ {*}) (x)) - f ^ {*} (x)\right) ^ {2} \mathbf {1} _ {x \in S} - \| \left(q \circ (\bar {\eta} (\mathbb {K} f ^ {*})) - f ^ {*}\right) \mathbf {1} _ {x \in S} \| _ {L ^ {2}} ^ {2} \right| \\ \lesssim \sqrt {\frac {\left(\sup  _ {z \in [ - 1 , 1 ]} | q (z) | ^ {2} + \iota^ {2 \ell}\right) \cdot \| (q \circ (\overline {{\eta}} (\mathbb {K} f ^ {*})) - f ^ {*}) \mathbf {1} _ {x \in S} \| _ {L ^ {2}} ^ {2} \iota}{n}} + \frac {\sup  _ {z \in [ - 1 , 1 ]} | q (z) | ^ {2} + \iota^ {2 \ell}}{n} \iota} \\ \lesssim \| (q \circ (\overline {{\eta}} (\mathbb {K} f ^ {*})) - f ^ {*}) \mathbf {1} _ {x \in S} \| _ {L ^ {2}} ^ {2} + \frac {\sup  _ {z \in [ - 1 , 1 ]} | q (z) | ^ {2} + \iota^ {2 \ell}}{n} \iota . \\ \end{array}
$$
Conditioning on the high probability event that  $x \in S$  for all  $x \in \mathcal{D}_2$ , we get that
$$
\begin{array}{l} \frac {1}{n} \sum_ {x \in \mathcal {D} _ {2}} \left(q (\overline {{\eta}} (\mathbb {K} f ^ {*}) (x)) - f ^ {*} (x)\right) ^ {2} \lesssim \| (q \circ (\overline {{\eta}} (\mathbb {K} f ^ {*})) - f ^ {*}) \mathbf {1} _ {x \in S} \| _ {L ^ {2}} ^ {2} + \frac {\sup _ {z \in [ - 1 , 1 ]} | q (z) | ^ {2} + \iota^ {2 \ell}}{n} \iota \\ \leq \| q \circ (\overline {{\eta}} (\mathbb {K} f ^ {*})) - f ^ {*} \| _ {L ^ {2}} ^ {2} + \frac {\sup _ {z \in [ - 1 , 1 ]} | q (z) | ^ {2} \iota + \iota^ {2 \ell + 1}}{n}. \\ \end{array}
$$
E Proofs for Section 4
E.1 Single Index Model
Proof of Theorem 2. It is easy to see that Assumptions 1 and 3 are satisfied. By assumption  $g^{*}$  is polynomially bounded, i.e. there exist constants  $C_g, \alpha_g$  such that
$$
g ^ {*} (z) \leq C _ {g} (1 + | z |) ^ {\alpha_ {g}} \leq C _ {g} 2 ^ {\alpha_ {g} - 1} (1 + | z | ^ {\alpha_ {g}}).
$$
$$
\| f ^ {*} \| _ {q} = \| g ^ {*} \| _ {L ^ {q} (\mathbb {R}, \mathcal {N} (0, 1)} \leq C _ {g} 2 ^ {\alpha_ {g} - 1} (1 + \mathbb {E} _ {z \sim \mathcal {N} (0, 1)} [ | z | ^ {\alpha_ {g} q} ] ^ {1 / q}) \leq C _ {g} 2 ^ {\alpha_ {g}} \alpha_ {g} ^ {\alpha_ {g} / 2} q ^ {\alpha_ {g} / 2}.
$$
Thus Assumption 2 is satisfied with  $C_f = C_g 2^{\alpha_g} \alpha_g^{\alpha_g / 2}$  and  $\ell = \alpha_g / 2$ .
Next, we see that
$$
K (x, x ^ {\prime}) = \mathbb {E} _ {v} [ (x \cdot v) (x ^ {\prime} \cdot v) ] = \frac {x \cdot x ^ {\prime}}{d}
$$