SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\mathcal {R} ^ {(j, x _ {0}, r)} \left(\mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}\right) \leq \mathcal {R} _ {n} ^ {(j, x _ {0}, r)} \left(\mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}\right) + \sqrt {\frac {\left| \mathcal {I} _ {(j , x _ {0} , r)} \right| \log (1 / \delta)}{2}}
$$
holds with probability at least  $1 - \delta$ .
In the following we use the notation:  $\hat{\mathbb{E}}_{S_{(j,x_0,r)}}\left[f_{\boldsymbol{w}}^{S_k}\right] = \frac{1}{\left|\mathcal{I}_{(j,x_0,r)}\right|}\sum_{i\in \mathcal{I}_{(j,x_0,r)}}f_{\boldsymbol{w}}^{S_k}(x_i)$ , then define:
$$
\Phi (S _ {(j, x _ {0}, r)}) = \sup  _ {f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x | y}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right] \right|.
$$
Lemma 4 Given  $\mathcal{B}(x_0,r)$ ,  $(x_0,y_0) \in S_j$  and  $f_{\boldsymbol{w}}^{S_k} \in \mathcal{F}_{\boldsymbol{w}}^{S_k}$ , with probability at least  $1 - \delta$ , the following holds for any  $f_{\boldsymbol{w}}^{S_k} \in \mathcal{F}_{\boldsymbol{w}}^{S_k}$ :
$$
\left| \mathbb {E} _ {x \sim \mathcal {D} _ {x | y}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right] \right| \leq \frac {2}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|} \mathcal {R} ^ {(j, x _ {0}, r)} (\mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}) + \sqrt {\frac {\log (1 / \delta)}{2 \left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}}
$$
Proof of Lemma 4 Similarly, let  $S_{(j,x_0,r)}'$  be a dataset that has at most one element different with  $S_{(j,x_0,r)}$ , then we have
$$
\begin{array}{l} \Phi \left(S _ {(j, x _ {0}, r)} ^ {\prime}\right) - \Phi \left(S _ {(j, x _ {0}, r)}\right) = \sup  _ {f _ {\boldsymbol {w}} ^ {S _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {S _ {k}}} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x | y}} \left[ f _ {\boldsymbol {w}} ^ {S _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)} ^ {\prime}} \left[ f _ {\boldsymbol {w}} ^ {S _ {k}} \right] \right| \\ - \sup  _ {f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x | y}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {\mathcal {S} _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right] \right| \\ \leq \sup  _ {f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}} \left| \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)} ^ {\prime}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right]} \right| \\ = \sup  _ {f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}} \left| \frac {1}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|} \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \left(f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \left(x _ {i} ^ {\prime}\right) - f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \left(x _ {i}\right)\right) \right| \\ \leq \frac {1}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}, \\ \end{array}
$$
then use McDiarmid's inequality, with probability at least  $1 - \delta$ , the following holds:
$$
\Phi \left(S _ {\left(j, x _ {0}, r\right)}\right) \leq \mathbb {E} _ {S _ {\left(j, x _ {0}, r\right)}} \left[ \Phi \left(S _ {\left(j, x _ {0}, r\right)}\right) \mid x _ {i} \in \mathcal {C} _ {\left(j, x _ {0}, r\right)} \right] + \sqrt {\frac {\log (1 / \delta)}{2 \left| \mathcal {I} _ {\left(j , x _ {0} , r\right)} \right|}}, \tag {1}
$$
then we could use the standard symmetrization technique ([56], Theorem 3.1) to derive the following:
$$
\begin{array}{l} \mathbb {E} _ {S _ {(j, x _ {0}, r)}} \left[ \Phi (S _ {(j, x _ {0}, r)}) \mid x _ {i} \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \\ = \mathbb {E} _ {S _ {(j, x _ {0}, r)}} \left[ \sup  _ {f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x | y}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right] \right| \mid x _ {i} \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \\ = \mathbb{E}_{S_{(j,x_{0},r)}}\left[\sup_{f_{\boldsymbol{w}}^{S_{k}}\in \mathcal{F}_{\boldsymbol{w}}^{S_{k}}}\left|\mathbb{E}_{S^{\prime}_{(j,x_{0},r)}}\left[\hat{\mathbb{E}}_{S^{\prime}_{(j,x_{0},r)}}\left[f_{\boldsymbol{w}}^{S_{k}}\right] - \hat{\mathbb{E}}_{S_{(j,x_{0},r)}}\left[f_{\boldsymbol{w}}^{S_{k}}\right]\mid x^{\prime}_{i}\in \mathcal{C}_{(j,x_{0},r)}\right]\right|\mid x_{i}\in \mathcal{C}_{(j,x_{0},r)}\right] \\ \end{array}
$$
$$
\begin{array}{l} \underset {J e n s e n} {\leq} \mathbb {E} _ {S _ {(j, x _ {0}, r)}, S _ {(j, x _ {0}, r)} ^ {\prime}} \left[ \sup  _ {f _ {\boldsymbol {w}} ^ {S _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {S _ {k}}} \left| \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)} ^ {\prime}} \left[ f _ {\boldsymbol {w}} ^ {S _ {k}} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w}} ^ {S _ {k}} \right] \right|   |   x _ {i}, x _ {i} ^ {\prime} \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \\ = \mathbb {E} _ {S _ {(j, x _ {0}, r)}, S _ {(j, x _ {0}, r)} ^ {\prime}} \left[ \sup  _ {f _ {\boldsymbol {w} ^ {k}} \in \mathcal {F} _ {\boldsymbol {w} ^ {k}} ^ {\mathcal {S} _ {k}}} \frac {1}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|} \left| \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \left(f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \left(x _ {i} ^ {\prime}\right) - f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \left(x _ {i}\right)\right) \right| | x _ {i}, x _ {i} ^ {\prime} \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \\ = \mathbb {E} _ {S _ {(j, x _ {0}, r)}, S _ {(j, x _ {0}, r)} ^ {\prime}} \left[ \sup  _ {f _ {\boldsymbol {w}} ^ {\boldsymbol {S} _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {\boldsymbol {S} _ {k}}} \frac {1}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|} \left| \mathbb {E} _ {\boldsymbol {\sigma}} \left[ \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \sigma_ {i} \left(f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \left(x _ {i} ^ {\prime}\right) - f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \left(x _ {i}\right)\right) \right] \right| | x _ {i}, x _ {i} ^ {\prime} \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \\ \end{array}
$$
(Since  $S$  and  $S^{\prime}$  have same distribution)
$$
\begin{array}{l} \underset {J e n s e n} {\leq} \mathbb {E} _ {S _ {(j, x _ {0}, r)}, S _ {(j, x _ {0}, r), \sigma} ^ {\prime}} \left[ \sup  _ {f _ {\boldsymbol {w}} ^ {S _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {S _ {k}}} \frac {1}{| \mathcal {I} _ {(j , x _ {0} , r)} |} \left| \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \sigma_ {i} \left(f _ {\boldsymbol {w}} ^ {S _ {k}} \left(x _ {i} ^ {\prime}\right) - f _ {\boldsymbol {w}} ^ {S _ {k}} (x _ {i})\right) \right| | x _ {i}, x _ {i} ^ {\prime} \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \\ \leq 2 \mathbb {E} _ {S _ {(j, x _ {0}, r)}, \boldsymbol {\sigma}} \left[ \sup  _ {f _ {\boldsymbol {w}} ^ {S _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {S _ {k}}} \frac {1}{| \mathcal {I} _ {(j , x _ {0} , r)} |} \left| \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \sigma_ {i} f _ {\boldsymbol {w}} ^ {S _ {k}} (x _ {i}) \right| | x _ {i} \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \\ = \frac {2}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|} \mathcal {R} ^ {(j, x _ {0}, r)} \left(\mathcal {F} _ {\boldsymbol {w}} ^ {S _ {k}}\right), \\ \end{array}
$$
combine with (1), we can get:
Proof of Lemma 1 Let  $\phi_k: \mathbb{R}^K \to \mathbb{R}$  satisfies  $\phi_k(v) = v_k$ , where  $v = (v_1, \ldots, v_k) \in \mathbb{R}^K$ . Then
$$
\mathcal {R} _ {n} ^ {(j, x _ {0}, r)} \left(\mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}\right) = \mathcal {R} _ {n} ^ {(j, x _ {0}, r)} \left(\phi_ {k} \circ \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S}}\right),
$$
since  $|\phi_k(v) - \phi_k(v')| = |v_k - v'_k| \leq \| v - v' \|_{\infty}, \forall v, v' \in \mathbb{R}^K$ , i.e.  $\phi_k$  is 1-Lipschitz. For  $f_{\pmb{w}}^S \in \mathcal{F}_{\pmb{w}}^{\mathcal{S}}$ , we have  $\| f_{\pmb{w}}^{\mathcal{S}} \|_{\infty} \leq 1$ , therefore,  $|\phi_k(f_{\pmb{w}}^{\mathcal{S}}(x))| \vee \| f_{\pmb{w}}^{\mathcal{S}}(x) \|_{\infty} \leq 1$ , then use Lemma 2, choose  $L = 1, \zeta = 1, a = \frac{1}{2}$ , there exists a constant  $C_k^{\pmb{w}} > 0$  such that
$$
\mathcal {R} _ {n} ^ {(j, x _ {0}, r)} \left(\phi_ {k} \circ \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S}}\right) \leq C _ {k} ^ {\boldsymbol {w}} \cdot \sqrt {K} \max  _ {k} \tilde {\mathcal {R}} _ {n} ^ {(j, x _ {0}, r)} \left(\mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}\right) \log^ {2} \left(\frac {\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}{\max  _ {k} \tilde {R} _ {n} ^ {(j , x _ {0} , r)} \left(\mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}\right)}\right), \tag {2}
$$
where  $\tilde{\mathcal{R}}_n^{(j,x_0,r)}\left(\mathcal{F}_{\boldsymbol{w}}^{\mathcal{S}_k}\right) = \sup_{S_{(j,x_0,r)}\in \mathcal{Z}_{(j,x_0,r)}^{\left|\mathcal{I}_{(j,x_0,r)}\right|}}\mathcal{R}_n^{(j,x_0,r)}\left(\mathcal{F}_{\boldsymbol{w}}^{\mathcal{S}_k}\right),$  and we denote
$$
\left\{(x, y) \in \mathcal {Z}: x \in \mathcal {C} _ {(j, x _ {0}, r)} \right\} \text {a s} \mathcal {Z} _ {(j, x _ {0}, r)}.
$$
For  $\forall S_{(j,x_0,r)}\in \mathcal{Z}_{(j,x_0,r)}^{\left|\mathcal{I}_{(j,x_0,r)}\right|},\forall f_{\boldsymbol{w}}^{S_k}\in \mathcal{F}_{\boldsymbol{w}}^{S_k}$ , we have
$$
\begin{array}{l} \mathbb {E} _ {\boldsymbol {\sigma}} \left[ \left| \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \sigma_ {i} f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x _ {i}) \right| \right] = \mathbb {E} _ {\boldsymbol {\sigma}} \left[ \sqrt {\left| \sum_ {i \in \mathcal {I} _ {(j , x _ {0} , r)}} \sigma_ {i} f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x _ {i}) \right| ^ {2}} \right] \\ \leq J e n s e n \sqrt {\mathbb {E} _ {\boldsymbol {\sigma}} \left[ \left| \sum_ {i \in \mathcal {I} _ {(j , x _ {0} , r)}} \sigma_ {i} f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x _ {i}) \right| ^ {2} \right]} \\ \leq \sqrt {\sum_ {i \in \mathcal {I} _ {(j , x _ {0} , r)}} \left| f _ {\boldsymbol {w}} ^ {S _ {k}} \left(x _ {i}\right) \right| ^ {2}} \leq \sqrt {n}, \\ \end{array}
$$
the penultimate inequality uses the Khintchine-Kahane Inequality [57]. Therefore,
$$
\mathcal {R} _ {n} ^ {(j, x _ {0}, r)} \left(\mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}\right) = \mathbb {E} _ {\boldsymbol {\sigma}} \left[ \sup  _ {f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}} \left| \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \sigma_ {i} f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x _ {i}) \right| \right] \leq \sqrt {n},
$$
which means  $\tilde{\mathcal{R}}_n^{(j,x_0,r)}\left(\mathcal{F}_{\boldsymbol{w}}^{\mathcal{S}_k}\right)\leq \sqrt{n}.$
Choose  $(\tilde{x},\tilde{y})\in \mathcal{Z}_{(j,x_0,r)},\tilde{f}_{\boldsymbol{w}}^{S_k}\in \tilde{\mathcal{F}}_{\boldsymbol{w}}^{S_k}$  satisfies
$$
\mathbb {E} _ {\boldsymbol {\sigma}} \left[ \left| \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \sigma_ {i} \tilde {f} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (\tilde {x}) \right| \right] = \sup  _ {z \in \mathcal {Z} _ {(j, x _ {0}, r)}, f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}} \mathbb {E} _ {\boldsymbol {\sigma}} \left[ \left| \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \sigma_ {i} f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \right| \right],
$$
then, similar to Lemma 3 in [58], we have
$$
\begin{array}{l} \sup_{S_{(j,x_{0},r)}\in \mathcal{Z}^{\left| \mathcal{I}_{(j,x_{0},r)}\right|}}\mathcal{R}_{n}^{(j,x_{0},r)}\left(\mathcal{F}_{\boldsymbol{w}}^{\mathcal{S}_{k}}\right) = \sup_{S_{(j,x_{0},r)}\in \mathcal{Z}^{\left| \mathcal{I}_{(j,x_{0},r)}\right|}}\mathbb{E}_{\boldsymbol{\sigma}}\left[\sup_{f_{\boldsymbol{w}}^{\mathcal{S}_{k}}\in \mathcal{F}_{\boldsymbol{w}}^{\mathcal{S}_{k}}}\left|\sum_{i\in \mathcal{I}_{(j,x_{0},r)}}\sigma_{i}f_{\boldsymbol{w}}^{\mathcal{S}_{k}}(x_{i})\right|\right] \\ \geq \sup  _ {z \in \mathcal {Z} _ {(j, x _ {0}, r)}} \mathbb {E} _ {\sigma} \left[ \sup  _ {f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \in \mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}} \left| \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \sigma_ {i} f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \right| \right] \\ \geq \sup_{Jensen}\sup_{z\in \mathcal{Z}_{(j,x_{0},r)},f_{\boldsymbol{w}}^{S_{k}}\in \mathcal{F}_{\boldsymbol{w}}^{S_{k}}}\mathbb{E}_{\boldsymbol{\sigma}}\left[\left|\sum_{i\in \mathcal{I}_{(j,x_{0},r)}}\sigma_{i}f_{\boldsymbol{w}}^{S_{k}}(x)\right|\right] \\ = \mathbb {E} _ {\boldsymbol {\sigma}} \left[ \left| \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \sigma_ {i} \tilde {f} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (\tilde {x}) \right| \right] \\ \geq \frac {\sqrt {2}}{2} \sqrt {\sum_ {i \in \mathcal {I} _ {(j , x _ {0} , r)}} \left| \tilde {f} _ {\boldsymbol {w}} ^ {S _ {k}} (\tilde {x}) \right| ^ {2}} \\ = \frac {\sqrt {2 \left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}}{2} \left| \tilde {f} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (\tilde {x}) \right|, \\ \end{array}
$$
similarly, the penultimate inequality also uses the Khintchine-Kahane Inequality [57]. Below we denote  $\left|\tilde{f}_{\boldsymbol{w}}^{\mathcal{S}_k}(\tilde{x})\right|$  as  $b^{w,k}$ . Choose  $b^{\boldsymbol{w}} = \max_{1\leq k\leq K}b^{\boldsymbol{w},k}$ , which satisfies  $\max_k\tilde{R}_n^{(j,x_0,r)}\geq \frac{\sqrt{2|\mathcal{I}_{(j,x_0,r)}|}b^{\boldsymbol{w}}}{2}$  and  $C^{\boldsymbol{w}} = \max_{1\leq k\leq K}C_{k}^{\boldsymbol{w}}$ , combine with (2), we have
$$
\mathcal {R} _ {n} ^ {(j, x _ {0}, r)} \left(\phi_ {k} \circ \mathcal {F} _ {\boldsymbol {w}} ^ {S}\right) \leq C ^ {\boldsymbol {w}} \cdot \sqrt {K \left| \mathcal {I} _ {(j , x _ {0} , r)} \right|} \log^ {2} \left(\frac {\sqrt {2 \left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}}{b ^ {\boldsymbol {w}}}\right) \tag {3}
$$
holds for all  $k \in [K]$ . Then use Lemma 3 and Lemma 4, with probability at least  $1 - \delta$ , holds
$$
\begin{array}{l} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x | y}} \left[ f _ {\pmb {w}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\pmb {w}} ^ {\mathcal {S} _ {k}} \right] \right| \\ \leq \frac {2}{| \mathcal {I} _ {(j , x _ {0} , r)} |} \mathcal {R} _ {n} ^ {(j, x _ {0}, r)} \left(\mathcal {F} _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}}\right) + 3 \sqrt {\frac {\log (2 / \delta)}{2 \left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}} \\ \end{array}
$$
then combine with (3), we have
$$
\begin{array}{l} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right] \right| \\ \leq 4 C ^ {\boldsymbol {w}} \sqrt {\frac {K}{| \mathcal {I} _ {(j , x _ {0} , r)} |}} \log^ {2} \left(\frac {\sqrt {2 | \mathcal {I} _ {(j , x _ {0} , r)} |}}{b ^ {\boldsymbol {w}}}\right) + 3 \sqrt {\frac {\log (2 / \delta)}{2 | \mathcal {I} _ {(j , x _ {0} , r)} |}} \\ \end{array}
$$
Proof of Theroem 1 For the inequality in Lemma 1, by taking  $\delta' = \frac{\delta}{K}$  for each  $k \in [K]$ , we can guarantee with probability at least  $1 - \delta$ , the inequality
$$
\begin{array}{l} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right] \right| \\ \leq 4 C ^ {\boldsymbol {w}} \sqrt {\frac {K}{| \mathcal {I} _ {(j , x _ {0} , r)} |}} \log^ {2} \left(\frac {\sqrt {2 | \mathcal {I} _ {(j , x _ {0} , r)} |}}{b ^ {\boldsymbol {w}}}\right) + 3 \sqrt {\frac {\log (2 K / \delta)}{2 | \mathcal {I} _ {(j , x _ {0} , r)} |}} \\ \end{array}
$$
simultaneously holds for each  $k \in [K]$ . Then for two different parametrized class  $\mathcal{F}_{\boldsymbol{w}_1}^{S_k} \coloneqq \left\{f_{\boldsymbol{w}_1}^{S_k} : \boldsymbol{w}_1 \in \mathcal{W}_1\right\}$  and  $\mathcal{F}_{\boldsymbol{w}}^{S_k} \coloneqq \left\{f_{\boldsymbol{w}}^{S_k} : \boldsymbol{w} \in \mathcal{W}\right\}$ , just take  $C = \max \left\{C^{\boldsymbol{w}_1}, C^{\boldsymbol{w}}\right\}$  and  $b = \min \left\{b^{\boldsymbol{w}_1}, b^{\boldsymbol{w}}\right\}$ , the following simultaneously holds for any  $i \in \{1, 2\}$  and  $k \in [K]$ :
$$
\begin{array}{l} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x | y}} \left[ f _ {\boldsymbol {w} _ {i}} ^ {\boldsymbol {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w} _ {i}} ^ {\boldsymbol {S} _ {k}} \right] \right| \\ \leq 4 C \sqrt {\frac {K}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}} \log^ {2} \left(\frac {\sqrt {2 \left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}}{b}\right) + 3 \sqrt {\frac {\log (2 K / \delta)}{2 \left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}}. \\ \end{array}
$$
According to the assumption  $\frac{1}{|\mathcal{I}_{(j,x_0,r)}|}\left|\sum_{i\in \mathcal{I}_{(j,x_0,r)}}\big(f_{\boldsymbol{w}_1}^{\mathcal{S}_k}(x_i) - f_{\boldsymbol{w}}^{\mathcal{S}_k}(x_i)\big)\right|\leq \gamma$  which means  $\left|\hat{\mathbb{E}}_{S_{(j,x_0,r)}}\left[f_{\boldsymbol{w}_1}^{\mathcal{S}_k}\right] - \hat{\mathbb{E}}_{S_{(j,x_0,r)}}\left[f_{\boldsymbol{w}}^{\mathcal{S}_k}\right]\right|\leq \gamma$  then we have
$$
\begin{array}{l} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w} _ {1}} ^ {S _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w}} ^ {S _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \right| \\ \leq \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x | y}} \left[ f _ {\boldsymbol {w} _ {1}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w} _ {1}} ^ {\mathcal {S} _ {k}} \right] \right| \\ \end{array}
$$
$$
\begin{array}{l} + \left| \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w} _ {1}} ^ {\mathcal {S} _ {k}} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right] \right. \\ + \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \hat {\mathbb {E}} _ {S _ {(j, x _ {0}, r)}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} \right] \right| \\ \leq 8 C \sqrt {\frac {K}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}} \log^ {2} \left(\frac {\sqrt {2 \left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}}{b}\right) + 6 \sqrt {\frac {\log (2 K / \delta)}{2 \left| \mathcal {I} _ {(j , x _ {0} , r)} \right|}} + \gamma , \\ \end{array}
$$
$$
\begin{array}{l} \left\| \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w} _ {1}} ^ {S} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w}} ^ {S} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \right\| _ {\infty} \\ = \max  _ {k \in [ K ]} \left| \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w} _ {1}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S} _ {k}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \right| \\ \leq 8 C \sqrt {\frac {K}{| \mathcal {I} _ {(j , x _ {0} , r)} |}} \log^ {2} \left(\frac {\sqrt {2 | \mathcal {I} _ {(j , x _ {0} , r)} |}}{b}\right) + 6 \sqrt {\frac {\log (2 K / \delta)}{2 | \mathcal {I} _ {(j , x _ {0} , r)} |}} + \gamma . \\ \end{array}
$$
By the definition, we have  $\left|\mathcal{I}_{(j,x_0,r)}\right| = \rho_{(j,x_0,r)}\cdot \mathrm{vol}\mathcal{B}(x_0,r)$ , and note that
$$
\left(\frac {2 r}{\sqrt {d}}\right) ^ {d} \leq \operatorname {v o l} \mathcal {B} (x _ {0}, r) = \frac {\pi^ {\frac {d}{2}}}{\Gamma (\frac {d}{2} + 1)} r ^ {d} \leq 6 r ^ {d},
$$
therefore we can derive that with probability at least  $1 - \delta$ , the following holds:
$$
\begin{array}{l} \left\| \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w} _ {1}} ^ {\mathcal {S}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] - \mathbb {E} _ {x \sim \mathcal {D} _ {x \mid y}} \left[ f _ {\boldsymbol {w}} ^ {\mathcal {S}} (x) \mid x \in \mathcal {C} _ {(j, x _ {0}, r)} \right] \right\| _ {\infty} \\ \leq 8 C \sqrt {\frac {K d ^ {d / 2}}{\rho_ {(j , x _ {0} , r)} 2 ^ {d} r ^ {d}}} \log^ {2} \left(\frac {2 r ^ {d} \sqrt {3 \rho_ {j} (x _ {0} , r)}}{b}\right) + 6 \sqrt {\frac {d ^ {d / 2} \log (2 K / \delta)}{\rho_ {(j , x _ {0} , r)} 2 ^ {d + 1} r ^ {d}}} + \gamma , \\ \end{array}
$$
$$
\begin{array}{l} \left\| R _ {(j, x _ {0}, r)} \left(\boldsymbol {w} _ {1}\right) - R _ {(j, x _ {0}, r)} \left(\boldsymbol {w} _ {2}\right) \right\| = \left\| \frac {1}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|} \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \left(\ell \left(\boldsymbol {w} _ {1}, x _ {i}\right) - \ell \left(\boldsymbol {w} _ {2}, x _ {i}\right)\right) \right\| \\ \leq \frac {1}{\left| \mathcal {I} _ {(j , x _ {0} , r)} \right|} \sum_ {i \in \mathcal {I} _ {(j, x _ {0}, r)}} \left\| \ell \left(\boldsymbol {w} _ {1}, x _ {i}\right) - \ell \left(\boldsymbol {w} _ {2}, x _ {i}\right) \right\| \\ \leq L _ {1} \left\| \boldsymbol {w} _ {1} - \boldsymbol {w} _ {2} \right\|, \\ \end{array}
$$
the second inequality holds because of Assumption 1. Then due to the Lipschitz continuous gradient of  $R_{(j,x_0,r)}(\boldsymbol{w})$ , we have
$$
\begin{array}{l} R _ {(j, x _ {0}, r)} (\boldsymbol {w} ^ {t + 1}) - R _ {(j, x _ {0}, r)} (\boldsymbol {w} ^ {t}) \leq \left\langle \nabla_ {\boldsymbol {w}} R _ {(j, x _ {0}, r)} (\boldsymbol {w} ^ {t}), \boldsymbol {w} ^ {t + 1} - \boldsymbol {w} ^ {t} \right\rangle + \frac {L _ {1}}{2} \left\| \boldsymbol {w} ^ {t + 1} - \boldsymbol {w} ^ {t} \right\| ^ {2} \\ = - \eta_ {t} \left\langle \nabla_ {\boldsymbol {w}} R _ {(j, x _ {0}, r)} (\boldsymbol {w} ^ {t}), \frac {1}{M} \sum_ {i \in \mathcal {I} _ {B ^ {t}}} \nabla_ {\boldsymbol {w}} \ell (\boldsymbol {w} ^ {t}, x _ {i}) \right\rangle \\ + \frac {L _ {1} \eta_ {t} ^ {2}}{2} \left\| \frac {1}{M} \sum_ {i \in \mathcal {I} _ {B t}} \nabla_ {\boldsymbol {w}} \ell (\boldsymbol {w} ^ {t}, x _ {i}) \right\| ^ {2} \\ \leq - \eta_ {t} \left\langle \nabla_ {\boldsymbol {w}} R _ {(j, x _ {0}, r)} (\boldsymbol {w} ^ {t}), \frac {1}{M} \sum_ {i \in \mathcal {I} _ {B ^ {t}}} \nabla_ {\boldsymbol {w}} \ell (\boldsymbol {w} ^ {t}, x _ {i}) \right\rangle + \frac {L _ {1} \eta_ {t} ^ {2} G ^ {2}}{2}, \\ \end{array}
$$
the inequality holds because of Assumption 2, prescribe notation  $\xi_{t} = \left|\mathcal{I}_{B^{t}}\cap \mathcal{I}_{(j,x_{0},r)}\right|$ , combined with Assumption 3, 4, we have