SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \left| \boldsymbol {s} _ {t, \boldsymbol {\theta} (\tau)} (\boldsymbol {x} (t)) \right| = \left| \frac {1}{\sqrt {m}} \sum_ {i = 1} ^ {m} \boldsymbol {\theta} _ {i} (\tau) \sigma \left(\boldsymbol {w} _ {i} ^ {\top} \boldsymbol {x} (t) + \boldsymbol {u} _ {i} ^ {\top} \boldsymbol {e} (t)\right) \right| \\ \leq \frac {1}{\sqrt {m}} \sum_ {i = 1} ^ {m} | \boldsymbol {\theta} _ {i} (\tau) | \left| \sigma \left(\boldsymbol {w} _ {i} ^ {\top} \boldsymbol {x} (t) + \boldsymbol {u} _ {i} ^ {\top} \boldsymbol {e} (t)\right) \right| \\ \leq \frac {1}{\sqrt {m}} \left(C _ {T, \delta} + C _ {T, e}\right) \sum_ {i = 1} ^ {m} | \boldsymbol {\theta} _ {i} (\tau) | \\ \leq \left(C _ {T, \delta} + C _ {T, e}\right) \| \boldsymbol {\theta} (\tau) \| _ {2} \\ \lesssim \left(C _ {T, \delta} + C _ {T, e}\right) \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m}\right), \tag {37} \\ \end{array}
$$
$$
\begin{array}{l} | \boldsymbol {f} _ {1} (\boldsymbol {x} (t)) | = | \boldsymbol {s} _ {t, \boldsymbol {\theta} (\tau)} (\boldsymbol {x} (t)) - \nabla_ {\boldsymbol {x} (t)} \log p _ {t} (\boldsymbol {x} (t)) | \\ \lesssim \left(C _ {T, \delta} + C _ {T, e}\right) \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m}\right) + C _ {T, \delta} ^ {\prime}, \\ \end{array}
$$
$$
\left| f _ {2} (\boldsymbol {x} (t)) \right| = \left| \sigma \left(\boldsymbol {w} ^ {\top} \boldsymbol {x} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)\right) \right| \lesssim C _ {T, \delta} + C _ {T, e},
$$
where  $C_{T,\delta}^{\prime} \coloneqq \max_{\boldsymbol{x}(t) \in [-C_{T,\delta}, C_{T,\delta}]^{d}} \left| \nabla_{\boldsymbol{x}(t)} \log p_{t}(\boldsymbol{x}(t)) \right|$ . This gives
$$
\widehat {\operatorname {R a d}} _ {n} (\mathcal {F} _ {t}) \lesssim (C _ {T, \delta} + C _ {T, e}) \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m} + 1\right) \left(\widehat {\operatorname {R a d}} _ {n} (\mathcal {F} _ {1, t}) + \widehat {\operatorname {R a d}} _ {n} (\mathcal {F} _ {2, t})\right).
$$
$$
\mathcal {F} _ {1, t} ^ {\prime} := \left\{\boldsymbol {s} _ {t, \boldsymbol {\theta} (\tau)} (\boldsymbol {x} (t)): \| \boldsymbol {\theta} (\tau) \| _ {2} \lesssim \left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m} \right\},
$$
according to Lemma 26.6 in [43], we get  $\widehat{\mathrm{Rad}}_n(\mathcal{F}_{1,t})\leq \widehat{\mathrm{Rad}}_n(\mathcal{F}_{1,t}')$ . Since
$$
\begin{array}{l} \widehat {n \operatorname {R a d}} _ {n} \left(\mathcal {F} _ {1, t} ^ {\prime}\right) = \mathbb {E} _ {\boldsymbol {\xi}} \left[ \sup  _ {\| \boldsymbol {\theta} (\tau) \| _ {2} \lesssim \left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m}} \sum_ {j = 1} ^ {n} \xi_ {j} \boldsymbol {s} _ {t, \boldsymbol {\theta} (\tau)} (\boldsymbol {x} _ {j} (t)) \right] \\ = \mathbb {E} _ {\boldsymbol {\xi}} \left[ \sup  _ {\| \boldsymbol {\theta} (\tau) \| _ {2} \lesssim \left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H} ^ {+} \sqrt {\tau / m}}} \sum_ {j = 1} ^ {n} \xi_ {j} \frac {1}{\sqrt {m}} \sum_ {i = 1} ^ {m} \boldsymbol {\theta} _ {i} (\tau) \sigma \left(\boldsymbol {w} _ {i} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} _ {i} ^ {\top} \boldsymbol {e} (t)\right) \right] \\ \leq \frac{1}{\sqrt{m}}\mathbb{E}_{\boldsymbol{\xi}}\left[\sup_{\substack{\| \boldsymbol {\theta}(\tau)\|_{2}\lesssim \bigl\|\boldsymbol{s}_{0,\boldsymbol{\theta}_{0}}\bigr\|_{\mathcal{H}} + \sqrt{\tau / m}\\ \| \boldsymbol{w}_{i}\|_{1} + \| \boldsymbol{u}_{i}\|_{1}\leq 1,i\in [n]}}\sum_{i = 1}^{m}\boldsymbol{\theta}_{i}(\tau)\sum_{j = 1}^{n}\xi_{j}\sigma (\boldsymbol{w}_{i}^{\top}\boldsymbol{x}_{j}(t) + \boldsymbol{u}_{i}^{\top}\boldsymbol {e}(t))\right] \\ \end{array}
$$
$$
\begin{array}{l} \leq \frac {1}{\sqrt {m}} \mathbb {E} _ {\boldsymbol {\xi}} \left[ \sup  _ {\| \boldsymbol {\theta} (\tau) \| _ {2} \lesssim \left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m}} \sum_ {i = 1} ^ {m} | \boldsymbol {\theta} _ {i} (\tau) | \sup  _ {\| \boldsymbol {w} _ {i} \| _ {1} + \| \boldsymbol {u} _ {i} \| _ {1} \leq 1, i \in [ n ]} \left| \sum_ {j = 1} ^ {n} \xi_ {j} \sigma \left(\boldsymbol {w} _ {i} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} _ {i} ^ {\top} \boldsymbol {e} (t)\right) \right| \right] \\ \leq \mathbb {E} _ {\boldsymbol {\xi}} \left[ \sup  _ {\| \boldsymbol {\theta} (\tau) \| _ {2} \lesssim \left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m}} \| \boldsymbol {\theta} (\tau) \| _ {2} \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \left| \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) \right| \right] \\ \lesssim \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m}\right) \mathbb {E} _ {\boldsymbol {\xi}} \left[ \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \left| \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) \right| \right], \\ \end{array}
$$
where the  $\{\xi_i\}_{i=1}^n$  are independent random variables with the distribution  $\mathbb{P}(\xi_i = 1) = \mathbb{P}(\xi_i = -1) = 1/2$ , and obviously,
$$
\sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) \geq \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\mathbf {0} _ {d} ^ {\top} \boldsymbol {x} _ {j} (t) + \mathbf {0} _ {d} ^ {\top} \boldsymbol {e} (t)) = 0,
$$
$$
\begin{array}{l} \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \left| \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) \right| \\ \leq \max  \left\{\sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)), \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \sum_ {j = 1} ^ {n} (- \xi_ {j}) \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) \right\} \\ \leq \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) + \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \sum_ {j = 1} ^ {n} (- \xi_ {j}) \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)), \\ \end{array}
$$
and by symmetry
$$
\begin{array}{l} \mathbb {E} _ {\boldsymbol {\xi}} \left[ \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \left| \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) \right| \right] \\ \leq \mathbb {E} _ {\boldsymbol {\xi}} \left[ \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) \right] + \mathbb {E} _ {\boldsymbol {\xi}} \left[ \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \sum_ {j = 1} ^ {n} (- \xi_ {j}) \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) \right] \\ = 2 \mathbb {E} _ {\boldsymbol {\xi}} \left[ \sup  _ {\| \boldsymbol {w} \| _ {1} + \| \boldsymbol {u} \| _ {1} \leq 1} \sum_ {j = 1} ^ {n} \xi_ {j} \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} _ {j} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) \right] = 2 n \widehat {\operatorname {R a d}} _ {n} (\mathcal {F} _ {2, t}), \\ \end{array}
$$
i.e.,  $\widehat{\mathrm{Rad}}_n(\mathcal{F}_{1,t})\leq \widehat{\mathrm{Rad}}_n(\mathcal{F}_{1,t}')\lesssim 2\left(\left\| s_{0,\pmb {\theta}_0}\right\|_{\mathcal{H}} + \sqrt{\tau / m}\right)\widehat{\mathrm{Rad}}_n(\mathcal{F}_{2,t})$  . According to Lemma 26.9 (contraction lemma) and Lemma 26.11 in [43], we have
$$
\widehat {\mathrm {R a d}} _ {n} (\mathcal {F} _ {2, t}) \leq (\| \boldsymbol {x} (t) \| _ {\infty} + \| \boldsymbol {e} (t) \| _ {\infty}) \sqrt {\frac {2 \log (4 d)}{n}} \lesssim (C _ {T, \delta} + C _ {T, \boldsymbol {e}}) \sqrt {\frac {\log (d + 1)}{n}}.
$$
Combining above, we obtain
$$
J _ {1} = \widehat {\mathrm {R a d}} _ {n} (\mathcal {F} _ {t}) \lesssim (C _ {T, \delta} + C _ {T, e}) ^ {2} \left(\left\| \pmb {s} _ {0, \pmb {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m} + 1\right) ^ {2} \sqrt {\frac {\log (d + 1)}{n}}.
$$
(ii) For  $J_{2}$ , by (36) and (37), we get
$$
\begin{array}{l} | \boldsymbol {f} (\boldsymbol {x} (t)) | = | \boldsymbol {s} _ {t, \boldsymbol {\theta} (\tau)} (\boldsymbol {x} (t)) - \nabla_ {\boldsymbol {x} (t)} \log p _ {t} (\boldsymbol {x} (t)) | | \sigma (\boldsymbol {w} ^ {\top} \boldsymbol {x} (t) + \boldsymbol {u} ^ {\top} \boldsymbol {e} (t)) | \\ \lesssim \left(C _ {T, \delta} + C _ {T, \boldsymbol {e}}\right) ^ {2} \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m}\right) + C _ {T, \delta} ^ {\prime} \left(C _ {T, \delta} + C _ {T, \boldsymbol {e}}\right), \\ \end{array}
$$
$$
J _ {2} \lesssim (C _ {T, \delta} + C _ {T, e}) ^ {2} \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m} + 1\right) \sqrt {\frac {\log (1 / \delta)}{n}}.
$$
(iii) For  $J_{3}$ , we similarly have
$$
\left| \boldsymbol {s} _ {t, \hat {\boldsymbol {\theta}} _ {n} (\tau)} (\boldsymbol {x} (t)) \right| \lesssim \left(C _ {T, \delta} + C _ {T, \boldsymbol {e}}\right) \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m}\right),
$$
hence by (36) and (37), we get
$$
\begin{array}{l} J _ {3} \leq \mathbb {E} _ {\boldsymbol {x} (t) \sim p _ {t} ^ {(n)}} \left| \left(\boldsymbol {s} _ {t, \boldsymbol {\theta} (\tau)} (\boldsymbol {x} (t)) - \boldsymbol {s} _ {t, \hat {\boldsymbol {\theta}} _ {n} (\tau)} (\boldsymbol {x} (t))\right) \sigma \left(\boldsymbol {w} _ {i} ^ {\top} \boldsymbol {x} (t) + \boldsymbol {u} _ {i} ^ {\top} \boldsymbol {e} (t)\right) \right| \\ \lesssim \left(C _ {T, \delta} + C _ {T, e}\right) ^ {2} \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\tau / m}\right). \\ \end{array}
$$
Combining (i), (ii) and (iii), we obtain
$$
\begin{array}{l} \frac {1}{m} \sum_ {i = 1} ^ {m} \| \hat {\boldsymbol {a}} _ {i} (\tau) - \boldsymbol {a} _ {i} (\tau) \| _ {2} ^ {2} \lesssim \frac {1}{m ^ {2}} \sum_ {i = 1} ^ {m} \left\| \int_ {0} ^ {\tau} \mathbb {E} _ {t \sim \mathcal {U} (0, T)} \left[ \lambda (t) (J _ {1} + J _ {2} + J _ {3}) \mathbf {1} _ {d} \right] d \tau_ {0} \right\| _ {2} ^ {2} \\ \lesssim \frac {1}{m ^ {2}} \sum_ {i = 1} ^ {m} \left\| \int_ {0} ^ {\tau} \left(J _ {1} + J _ {2} + J _ {3}\right) \mathbf {1} _ {d} d \tau_ {0} \right\| _ {2} ^ {2} \\ = \left(J _ {1} + J _ {2} + J _ {3}\right) ^ {2} \tau^ {2} \frac {d}{m} \\ \lesssim \tau^ {2} \frac {d}{m} \left(C _ {T, \delta} + C _ {T, e}\right) ^ {4} \left[ \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} ^ {2} + \frac {\tau}{m} + 1\right) ^ {2} \right. \\ \left. \cdot \left(\sqrt {\frac {\log (d + 1)}{n}} + \sqrt {\frac {\log (1 / \delta)}{n}}\right) ^ {2} + \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} ^ {2} + \frac {\tau}{m}\right) \right], \\ \end{array}
$$
$$
I _ {1} = \tilde {\mathcal {L}} (\hat {\boldsymbol {\theta}} _ {n} (\tau)) - \tilde {\mathcal {L}} (\boldsymbol {\theta} (\tau))
$$
$$
\begin{array}{l} \lesssim \left(C _ {T, \delta} + C _ {T, e}\right) \left(\sqrt {\tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})} + \frac {1}{\sqrt {\tau}} \left(\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \| _ {\mathcal {H}} + \| \boldsymbol {s} _ {0, \boldsymbol {\theta} ^ {*}} \| _ {\mathcal {H}}\right)\right) \left\{\frac {1}{m} \sum_ {i = 1} ^ {m} \| \hat {\boldsymbol {a}} _ {i} (\tau) - \boldsymbol {a} _ {i} (\tau) \| _ {2} ^ {2} \right\} ^ {\frac {1}{2}} \\ + \left(C _ {T, \delta} ^ {2} + C _ {T, e} ^ {2}\right) \frac {1}{m} \sum_ {i = 1} ^ {m} \| \hat {\boldsymbol {a}} _ {i} (\tau) - \boldsymbol {a} _ {i} (\tau) \| _ {2} ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \lesssim \left(C _ {T, \delta} + C _ {T, \boldsymbol {e}}\right) ^ {3} \left(\sqrt {\tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})} + \frac {1}{\sqrt {\tau}} \left(\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \| _ {\mathcal {H}} + \| \boldsymbol {s} _ {0, \boldsymbol {\theta} ^ {*}} \| _ {\mathcal {H}}\right)\right) \\ \cdot \tau \sqrt {\frac {d}{m}} \left[ \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} ^ {2} + \tau + 1\right) \left(\sqrt {\frac {\log (d + 1)}{n}} + \sqrt {\frac {\log (1 / \delta)}{n}}\right) + \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} + \sqrt {\frac {\tau}{m}}\right) \right] \\ + \tau^ {2} \frac {d}{m} \left(C _ {T, \delta} + C _ {T, \boldsymbol {e}}\right) ^ {6} \left[ \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} ^ {2} + \tau + 1\right) ^ {2} \left(\sqrt {\frac {\log (d + 1)}{n}} + \sqrt {\frac {\log (1 / \delta)}{n}}\right) ^ {2} + \left(\left\| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \right\| _ {\mathcal {H}} ^ {2} + \frac {\tau}{m}\right) \right] \\ \lesssim \left(C _ {T, \delta} + C _ {T, \mathbf {e}}\right) ^ {6} \tau \sqrt {\frac {d}{m}} \left[ (\tau + 1) \left(\sqrt {\frac {\log (d + 1)}{n}} + \sqrt {\frac {\log (1 / \delta)}{n}}\right) + \left(\frac {\| \mathbf {A} _ {0} \| _ {F}}{\sqrt {m}} + \sqrt {\frac {\tau}{m}}\right) \right] \\ \cdot \left\{\left(\sqrt {\tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})} + \frac {1}{\sqrt {m \tau}} \left(\| \boldsymbol {A} _ {0} \| _ {F} + \| \boldsymbol {A} _ {*} \| _ {F}\right)\right) \right. \\ + \tau \sqrt {\frac {d}{m}} \left[ (\tau + 1) \left(\sqrt {\frac {\log (d + 1)}{n}} + \sqrt {\frac {\log (1 / \delta)}{n}}\right) + \left(\frac {\| \boldsymbol {A} _ {0} \| _ {F}}{\sqrt {m}} + \sqrt {\frac {\tau}{m}}\right) \right] \Bigg \} \tag {38} \\ \end{array}
$$
$$
\begin{array}{l} \lesssim \log^ {3} (1 / \delta^ {2}) \tau \sqrt {\frac {d}{m}} \left[ (\tau + 1) \left(\sqrt {\frac {\log (d + 1)}{n}} + \sqrt {\frac {\log (1 / \delta)}{n}}\right) + \left(\frac {1}{\sqrt {m}} + \sqrt {\frac {\tau}{m}}\right) \right] \\ \cdot \left\{\left(\sqrt {\tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})} + \frac {1}{\sqrt {m \tau}}\right) + \tau \sqrt {\frac {d}{m}} \left[ (\tau + 1) \left(\sqrt {\frac {\log (d + 1)}{n}} + \sqrt {\frac {\log (1 / \delta)}{n}}\right) + \left(\frac {1}{\sqrt {m}} + \sqrt {\frac {\tau}{m}}\right) \right] \right\}, \\ \end{array}
$$
Combining all above estimates yields
$$
\begin{array}{l} D _ {\mathrm {K L}} \left(p _ {0} \| p _ {0, \hat {\boldsymbol {\theta}} _ {n} (\tau)}\right) \\ \lesssim I _ {1} + I _ {2} + I _ {3} + D _ {\mathrm {K L}} \left(p _ {T} \| \pi\right) \\ \lesssim \log^ {3} (1 / \delta^ {2}) \tau \sqrt {\frac {d}{m}} \left[ (\tau + 1) \left(\sqrt {\frac {\log (d + 1)}{n}} + \sqrt {\frac {\log (1 / \delta)}{n}}\right) + \frac {1}{\sqrt {m}} \left(1 + \sqrt {\tau}\right) \right] \\ \cdot \left\{\left(\sqrt {\tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})} + \frac {1}{\sqrt {m \tau}}\right) + \tau \sqrt {\frac {d}{m}} \left[ (\tau + 1) \left(\sqrt {\frac {\log (d + 1)}{n}} + \sqrt {\frac {\log (1 / \delta)}{n}}\right) + \frac {1}{\sqrt {m}} (1 + \sqrt {\tau}) \right] \right\} \\ \left. \right. + \left(\bar {\tilde {\mathcal {L}}} \left(\bar {\boldsymbol {\theta}} ^ {*}\right) + \tilde {\mathcal {L}} \left(\boldsymbol {\theta} ^ {*}\right)\right) + \frac {\log^ {2} (1 / \delta^ {2})}{m} d + \frac {1}{\tau} \left(\left\| \bar {\boldsymbol {s}} _ {0, \bar {\boldsymbol {\theta}} _ {0}} \right\| _ {\mathcal {H}} ^ {2} + \left\| \bar {\boldsymbol {s}} _ {0, \bar {\boldsymbol {\theta}} ^ {*}} \right\| _ {\mathcal {H}} ^ {2} + \| \boldsymbol {s} _ {0, \boldsymbol {\theta} _ {0}} \| _ {\mathcal {H}} ^ {2} + \| \boldsymbol {s} _ {0, \boldsymbol {\theta} ^ {*}} \| _ {\mathcal {H}} ^ {2}\right) + D _ {\mathrm {K L}} \left(p _ {T} \| \pi\right). \\ \end{array}
$$
If one only focuses on the  $(m,n)$ -dependence, i.e. the dependence on the model capacity and sample size, this upper bound can be further simplified as
$$
\begin{array}{l} D _ {\mathrm {K L}} \left(p _ {0} \| p _ {0, \hat {\boldsymbol {\theta}} _ {n} (\tau)}\right) \\ \stackrel {\text {(v i)}} {\lesssim} \left(\frac {\tau^ {2}}{\sqrt {m n}} + \frac {\tau \sqrt {\tau}}{m}\right) \cdot \left(\sqrt {\tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})} + \frac {1}{\sqrt {m \tau}} + \frac {\tau^ {2}}{\sqrt {m n}} + \frac {\tau \sqrt {\tau}}{m}\right) \\ + \left(\bar {\tilde {\mathcal {L}}} (\bar {\boldsymbol {\theta}} ^ {*}) + \tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})\right) + \frac {1}{m} + \frac {1}{\tau} + D _ {\mathrm {K L}} (p _ {T} \| \pi) \\ \leq \left(\sqrt {\tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})} + \frac {1}{\sqrt {m \tau}} + \frac {\tau^ {2}}{\sqrt {m n}} + \frac {\tau \sqrt {\tau}}{m}\right) ^ {2} + \left(\bar {\tilde {\mathcal {L}}} (\bar {\boldsymbol {\theta}} ^ {*}) + \tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})\right) + \frac {1}{m} + \frac {1}{\tau} + D _ {\mathrm {K L}} (p _ {T} \| \pi) \\ \lesssim \left(\tilde {\mathcal {L}} \left(\boldsymbol {\theta} ^ {*}\right) + \frac {1}{m \tau} + \frac {\tau^ {4}}{m n} + \frac {\tau^ {3}}{m ^ {2}}\right) + \left(\bar {\tilde {\mathcal {L}}} \left(\bar {\boldsymbol {\theta}} ^ {*}\right) + \tilde {\mathcal {L}} \left(\boldsymbol {\theta} ^ {*}\right)\right) + \frac {1}{m} + \frac {1}{\tau} + D _ {\mathrm {K L}} \left(p _ {T} \| \pi\right) \\ \lesssim \frac {\tau^ {4}}{m n} + \frac {\tau^ {3}}{m ^ {2}} + \frac {1}{\tau} + \frac {1}{m} + \left(\bar {\tilde {\mathcal {L}}} (\bar {\boldsymbol {\theta}} ^ {*}) + \tilde {\mathcal {L}} (\boldsymbol {\theta} ^ {*})\right) + D _ {\mathrm {K L}} (p _ {T} \| \pi), \\ \end{array}
$$
where we assume  $\tau \geq 1$  for simplicity, and  $\stackrel{\mathrm{(vi)}}{\lesssim}$  hides the term  $d\log (d + 1)$ , the polynomials of  $\log (1 / \delta^2)$  and finite RKHS norms  $\| \cdot \|_{\mathcal{H}}$ . The proof is completed.
Approximation errors. For the (universal) approximation, we discuss in two points:
- First, the approximation is a separate problem that can be analyzed independently of training and generalization. While it is beyond the scope of current work, we have included the approximation error in the final estimates. When the approximation by random feature models fails, the generalization error is supposed to be significant.
- In addition, the random feature model can approximate Lipschitz continuous functions on a compact domain (Theorem 6 in [17]). Notice that the forward diffusion process defines a random path  $(\pmb{x}(t), t)_{t \in [0,T]}$  contained in a rectangular domain  $R_{T,\delta} \coloneqq [-C_{T,\delta}, C_{T,\delta}]^d \times [0,T] \subset \mathbb{R}^{d+1}$  with  $C_{T,\delta} \coloneqq C_T(C_x + \sqrt{\log(1/\delta^2)})$  (use Lemma 1 and the boundness of inputs), one can apply Theorem 6 in [17] to bound (7) on the domain  $R_{T,\delta}$  in  $\mathbb{R}^{d+1}$  to obtain approximation results for Lipschitz continuous target score functions.
A.2 Data-Dependent Generalization Gap
Lemma 7 (Forward perturbation estimates, Gaussian mixtures). Suppose that  $x$  is sampled from a one-dimensional 2-mode Gaussian mixture:  $p_0(x) = q_1\mathcal{N}(x; - \mu ,1) + q_2\mathcal{N}(x;\mu ,1)$ , where  $\mu >0$ ,  $q_{1}$ ,  $q_{2} > 0$  with  $q_{1} + q_{2} = 1$  are all constants. Then for any  $\delta >0$ ,  $\delta \ll 1$ ,  $\mu >\sqrt{\log(1 / \delta^2)}$ , with the probability of at least  $1 - \delta$ , we have
$$
| x (t) | \lesssim C _ {T} \left(\mu + \sqrt {\log \left(1 / \delta^ {2}\right)}\right) \triangleq C _ {T, \mu , \delta}. \tag {39}
$$
Proof. It is straightforward to verify that
$$
\begin{array}{l} \mathbb {P} \left(\left\{x: | x - \mu | \leq \sqrt {\log \left(1 / \delta^ {2}\right)} \right\} \cup \left\{x: | x + \mu | \leq \sqrt {\log \left(1 / \delta^ {2}\right)} \right\}\right) \\ = \mathbb {P} \{x: | x - \mu | \leq \sqrt {\log (1 / \delta^ {2})} \} + \mathbb {P} \{x: | x + \mu | \leq \sqrt {\log (1 / \delta^ {2})} \} \\ = \int_ {\mu - \sqrt {\log (1 / \delta^ {2})}} ^ {\mu + \sqrt {\log (1 / \delta^ {2})}} p _ {0} (x) d x + \int_ {- \mu - \sqrt {\log (1 / \delta^ {2})}} ^ {- \mu + \sqrt {\log (1 / \delta^ {2})}} p _ {0} (x) d x \\ \geq q _ {2} \int_ {\mu - \sqrt {\log (1 / \delta^ {2})}} ^ {\mu + \sqrt {\log (1 / \delta^ {2})}} \mathcal {N} (x; \mu , 1) d x + q _ {1} \int_ {- \mu - \sqrt {\log (1 / \delta^ {2})}} ^ {- \mu + \sqrt {\log (1 / \delta^ {2})}} \mathcal {N} (x; - \mu , 1) d x \\ = q _ {2} \int_ {- \sqrt {\log (1 / \delta^ {2})}} ^ {\sqrt {\log (1 / \delta^ {2})}} \mathcal {N} (x; 0, 1) d x + q _ {1} \int_ {- \sqrt {\log (1 / \delta^ {2})}} ^ {\sqrt {\log (1 / \delta^ {2})}} \mathcal {N} (x; 0, 1) d x \\ = \left(q _ {1} + q _ {2}\right) \cdot \mathbb {P} \left\{\epsilon : | \epsilon | \leq \sqrt {\log \left(1 / \delta^ {2}\right)} \right\} \geq 1 - \delta , \\ \end{array}
$$
where the last inequality applies (21). That is, for any  $\delta > 0$ ,  $\delta \ll 1$ , with the probability of at least  $1 - \delta$ , we have
$$
| x | \in [ \mu - \sqrt {\log (1 / \delta^ {2})}, \mu + \sqrt {\log (1 / \delta^ {2})} ] = \Theta (\mu). \tag {40}
$$
Hence, Lemma 1 gives
$$
\left| x (t) \right| \lesssim C _ {T} \left(\mu + \sqrt {\log \left(1 / \delta^ {2}\right)}\right), \tag {41}
$$
Lemma 8 (Monte Carlo estimates, Gaussian mixtures). Define the Monte Carlo error
$$
\operatorname {E r r} _ {\mathrm {M C}} = \operatorname {E r r} _ {\mathrm {M C}} (\boldsymbol {\theta}, \bar {\boldsymbol {\theta}}; T, \lambda (\cdot)) := \mathbb {E} _ {t \sim \mathcal {U} (0, T)} [ \lambda (t) \cdot \mathbb {E} _ {x (t) \sim p _ {t}} [ \| s _ {t, \boldsymbol {\theta}} (x (t)) - \bar {s} _ {t, \bar {\boldsymbol {\theta}}} (x (t)) \| _ {2} ^ {2} ] ]. \tag {42}
$$
Suppose that the trainable parameter  $\mathbf{a}$  and embedding function  $e(\cdot)$  are both bounded, and  $x$  is sampled from a one-dimensional 2-mode Gaussian mixture:  $p_0(x) = q_1\mathcal{N}(x; - \mu ,1) + q_2\mathcal{N}(x;\mu ,1)$ , where  $\mu >0$ ,  $q_{1}$ ,  $q_{2} > 0$  with  $q_{1} + q_{2} = 1$  are all constants. Then, given any  $\bar{\theta}$ , for any  $\delta >0$ ,  $\delta \ll 1$ ,  $\mu >\sqrt{\log(1 / \delta^2)}$ , with the probability of at least  $1 - \delta$ , there exists  $\pmb{\theta}$  such that
$$
\operatorname {E r r} _ {\mathrm {M C}} \lesssim \mu^ {2} \frac {\log (1 / \delta)}{m}, \tag {43}
$$
Proof. According to Lemma 7, we just need to follow the proof of Lemma 6 by replacing  $C_{T,\delta}$  by  $C_{T,\mu,\delta}$ . Notably, based on (33) in the proof of Lemma 6, one can finally derive
$$
\mathrm {E r r} _ {\mathrm {M C}} \lesssim \mu^ {2} \frac {\log (1 / \delta)}{m},
$$
which gives the desired estimates.
Proof of Theorem 2. We decompose the loss  $\tilde{\mathcal{L}} (\hat{\theta}_n(\tau))$  in the same way as in the proof of Theorem 1. In fact, Theorem 2 is similarly proved by replacing  $C_{T,\delta}$  in the proof of Theorem 1 by  $C_{T,\mu ,\delta}$ . Note that  $C_{T,\mu ,\delta} = C_T\left(\mu +\sqrt{\log(1 / \delta^2)}\right)\lesssim \mu$  for  $\mu \gg 1$ , the proof is completed.