SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\mathcal {G} _ {T} = \{g (\boldsymbol {z}) = \sum_ {i = 1} ^ {n} - \frac {1}{n} \mathsf {K} (\boldsymbol {z}, \boldsymbol {z} _ {i} ^ {\prime}; \mathcal {S} ^ {\prime}) + \ell (\boldsymbol {w} _ {0}, \boldsymbol {z}): \mathsf {K} (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T} \}.
$$
$$
\mathcal {K} _ {T} = \{\mathsf {K} _ {T} (\cdot , \cdot ; \mathcal {S} ^ {\prime}): \mathcal {S} ^ {\prime} \in \mathsf {s u p p} (\mu^ {\otimes n}), \frac {1}{n ^ {2}} \sum_ {i, j} \mathsf {K} _ {T} (\boldsymbol {z} _ {i} ^ {\prime}, \boldsymbol {z} _ {j} ^ {\prime}; \mathcal {S} ^ {\prime}) \leq B ^ {2} \}.
$$
Suppose  $\mathsf{K}(z,z^{\prime};\mathcal{S}^{\prime}) = \langle \Phi_{\mathcal{S}^{\prime}}(z),\Phi_{\mathcal{S}^{\prime}}(z^{\prime})\rangle$  . Define
$$
\mathcal {G} _ {T} ^ {U} = \left\{g (\boldsymbol {z}) = \left\langle \beta , \Phi_ {\mathcal {S} ^ {\prime}} (\boldsymbol {z}) \right\rangle + \ell \left(\boldsymbol {w} _ {0}, \boldsymbol {z}\right): \| \boldsymbol {\beta} \| \leq B, K (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T} \right\}.
$$
We first show  $\mathcal{G}_T\subseteq \mathcal{G}_T^U$  .For  $\forall g(z)\in \mathcal{G}_T$
$$
\begin{array}{l} g (\boldsymbol {z}) = \sum_ {i = 1} ^ {n} - \frac {1}{n} K (\boldsymbol {z}, \boldsymbol {z} _ {i} ^ {\prime}; \mathcal {S} ^ {\prime}) + \ell (\boldsymbol {w} _ {0}, \boldsymbol {z}) \\ = \sum_ {i = 1} ^ {n} - \frac {1}{n} \left\langle \Phi_ {\mathcal {S} ^ {\prime}} (\boldsymbol {z}), \Phi_ {\mathcal {S} ^ {\prime}} \left(\boldsymbol {z} _ {i} ^ {\prime}\right) \right\rangle + \ell \left(\boldsymbol {w} _ {0}, \boldsymbol {z}\right) \quad \left(\text {b y d e f i n i t i o n} K (\boldsymbol {z}, \boldsymbol {z} ^ {\prime}; \mathcal {S} ^ {\prime}) = \left\langle \Phi_ {\mathcal {S} ^ {\prime}} (\boldsymbol {z}), \Phi_ {\mathcal {S} ^ {\prime}} \left(\boldsymbol {z} ^ {\prime}\right) \right\rangle\right) \\ = \left\langle \Phi_ {\mathcal {S} ^ {\prime}} (\boldsymbol {z}), \sum_ {i = 1} ^ {n} - \frac {1}{n} \Phi_ {\mathcal {S} ^ {\prime}} \left(\boldsymbol {z} _ {i} ^ {\prime}\right) \right\rangle + \ell \left(\boldsymbol {w} _ {0}, \boldsymbol {z}\right) \\ = \left\langle \Phi_ {\mathcal {S} ^ {\prime}} (\boldsymbol {z}), \beta_ {\mathcal {S} ^ {\prime}} \right\rangle + \ell (\boldsymbol {w} _ {0}, \boldsymbol {z}) \quad \left(\text {d e n o t e} \beta_ {\mathcal {S} ^ {\prime}} = \sum_ {i = 1} ^ {n} - \frac {1}{n} \Phi_ {\mathcal {S} ^ {\prime}} \left(\boldsymbol {z} _ {i} ^ {\prime}\right)\right) \\ = \left\langle \beta_ {\mathcal {S} ^ {\prime}}, \Phi_ {\mathcal {S} ^ {\prime}} (\boldsymbol {z}) \right\rangle + \ell (\boldsymbol {w} _ {0}, \boldsymbol {z}) \\ \end{array}
$$
We know by definition of  $\mathcal{G}_T$ ,  $\| \beta_{\mathcal{S}'}\|^2 = \frac{1}{n^2}\sum_{i,j}\mathsf{K}(z_i',z_j';\mathcal{S}')\leq B^2$ . Thus  $g(z)\in \mathcal{G}_T^U$ . Since  $\forall g(z)\in \mathcal{G}_T$ ,  $g(z)\in \mathcal{G}_T^U$ ,  $\mathcal{G}_T\subseteq \mathcal{G}_T^U$ . But  $\mathcal{G}_T^U$  is strictly larger than  $\mathcal{G}_T$  because  $\beta_{\mathcal{S}'}$  is a fixed vector for a fixed  $\mathsf{K}(\cdot ,\cdot ;\mathcal{S}')$  while  $\beta$  in  $\mathcal{G}_T^U$  is a vector of any direction.
The second term in the square root is
Then by the property of Rademacher complexity,
$$
\begin{array}{l} \underset {\boldsymbol {\sigma}} {\mathbb {E}} \left[ \sup  _ {\mathsf {K} (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T}} \sum_ {i \neq j} \sigma_ {i} \sigma_ {j} \mathsf {K} (\boldsymbol {z} _ {i}, \boldsymbol {z} _ {j}; \mathcal {S} ^ {\prime}) \right] \\ \leq \underset {\boldsymbol {\sigma}} {\mathbb {E}} \left[ \sum_ {i \neq j} \sup  _ {K (\cdot , \cdot ; S ^ {\prime}) \in \mathcal {K} _ {T}} \sigma_ {i} \sigma_ {j} K (z _ {i}, z _ {j}; S ^ {\prime}) \right] \\ = \sum_ {i \neq j} \mathbb {E} _ {\boldsymbol {\sigma}} \left[ \sup  _ {\mathsf {K} (\cdot ,; S ^ {\prime}) \in \mathcal {K} _ {T}} \sigma_ {i} \sigma_ {j} \mathsf {K} (\boldsymbol {z} _ {i}, \boldsymbol {z} _ {j}; S ^ {\prime}) \right] \\ = \sum_ {i \neq j} \left(\mathbb {P} \left(\sigma_ {i} \sigma_ {j} = + 1\right) \left[ \sup  _ {\mathsf {K} (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T}} \mathsf {K} \left(\boldsymbol {z} _ {i}, \boldsymbol {z} _ {j}; \mathcal {S} ^ {\prime}\right) \right] + \mathbb {P} \left(\sigma_ {i} \sigma_ {j} = - 1\right) \left[ \sup  _ {\mathsf {K} (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T}} - \mathsf {K} \left(\boldsymbol {z} _ {i}, \boldsymbol {z} _ {j}; \mathcal {S} ^ {\prime}\right) \right]\right) \\ = \sum_ {i \neq j} \frac {1}{2} \left[ \sup  _ {\mathsf {K} (\cdot ,; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T}} \mathsf {K} (\boldsymbol {z} _ {i}, \boldsymbol {z} _ {j}; \mathcal {S} ^ {\prime}) - \inf  _ {\mathsf {K} (\cdot ,; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T}} \mathsf {K} (\boldsymbol {z} _ {i}, \boldsymbol {z} _ {j}; \mathcal {S} ^ {\prime}) \right] \\ = \sum_ {i \neq j} \Delta \left(\boldsymbol {z} _ {i}, \boldsymbol {z} _ {j}\right) \\ \end{array}
$$
where we define  $\Delta (\pmb {z}_i,\pmb {z}_j) = \frac{1}{2}\left[\sup_{\mathsf{K}(\cdot ,\cdot ;\mathcal{S}^{\prime})\in \mathcal{K}_T}\mathsf{K}(\pmb {z}_i,\pmb {z}_j;\mathcal{S}^{\prime}) - \inf_{\mathsf{K}(\cdot ,\cdot ;\mathcal{S}^{\prime})\in \mathcal{K}_T}\mathsf{K}(\pmb {z}_i,\pmb {z}_j;\mathcal{S}^{\prime})\right].$  Thus in total,
$$
\hat {\mathcal {R}} _ {\mathcal {S}} \left(\mathcal {G} _ {T}\right) \leq \hat {\mathcal {R}} _ {\mathcal {S}} \left(\mathcal {G} _ {T} ^ {U}\right) \leq \frac {B}{n} \left(\sup  _ {\mathsf {K} (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T}} \operatorname {T r} \left(\mathsf {K} \left(\mathbf {Z}, \mathbf {Z}; \mathcal {S} ^ {\prime}\right)\right) + \sum_ {i \neq j} \Delta \left(\boldsymbol {z} _ {i}, \boldsymbol {z} _ {j}\right)\right) ^ {\frac {1}{2}}
$$
C.2.2 Proof of  $U_{2}$
Proof. To simplify the notation, denote  $\mathsf{K}_{\mathcal{S}'}(\cdot, \cdot) = \mathsf{K}_T(\cdot, \cdot; \mathcal{S}')$ . Denote  $g_{\mathsf{K}_{\mathcal{S}'}}(z) = \sum_{i=1}^{n} -\frac{1}{n}\mathsf{K}_{\mathcal{S}'}(z, z_i') + \ell(\boldsymbol{w}_0, z)$  for a fixed  $\mathsf{K}_{\mathcal{S}'}$ , which is a singleton hypothesis class. Then  $\mathcal{G}_T$  is a union set of such function classes with different kernels,
$$
\mathcal {G} _ {T} = \left\{g _ {\mathsf {K} _ {S ^ {\prime}}} (\boldsymbol {z}) = \sum_ {i = 1} ^ {n} - \frac {1}{n} \mathsf {K} _ {S ^ {\prime}} (\boldsymbol {z}, \boldsymbol {z} _ {i} ^ {\prime}) + \ell (\boldsymbol {w} _ {0}, \boldsymbol {z}): \mathsf {K} _ {S ^ {\prime}} \in \mathcal {K} _ {T} \right\} = \bigcup_ {\mathsf {K} _ {S ^ {\prime}} \in \mathcal {K} _ {T}} \left\{g _ {\mathsf {K} _ {S ^ {\prime}}} (\boldsymbol {z}) \right\}
$$
Then we can rewrite  $\mathcal{G}_T^S = \{g_{\mathsf{K}_{S'}}(\mathbf{Z}) = (g_{\mathsf{K}_{S'}}(\pmb {z}_1),\dots ,g_{\mathsf{K}_{S'}}(\pmb {z}_n)):K_{S'}\in \mathcal{K}_T\}$ . Suppose  $\mathcal{G}_T^\epsilon \subseteq \mathcal{G}_T^S$  is a minimal  $\epsilon$ -cover of  $\mathcal{G}_T^S$  with  $\| \cdot \| _1$ . Denote  $\widetilde{g}_{\mathsf{K}_{S'}}(\mathbf{Z}) = (\widetilde{g}_{\mathsf{K}_{S'}}(\pmb {z}_1),\dots ,\widetilde{g}_{\mathsf{K}_{S'}}(\pmb {z}_n))\in \mathcal{G}_T^\epsilon$  as the closest element to  $g_{\mathsf{K}_{S'}}(\mathbf{Z})\in \mathcal{G}_T^S$  and  $\widetilde{g}_{\mathsf{K}_{S'}}(\pmb {z}) = \sum_{i = 1}^{n} - \frac{1}{n}\widetilde{K}_{\mathsf{S}'}(\pmb {z},\pmb {z}_i') + \ell (\pmb {w}_0,\pmb {z}) = g_{\widetilde{K}_{\mathsf{S}'}}(z)$  with  $\widetilde{K}_{\mathcal{S}'}\in \mathcal{K}_T$ . Denote the set of all  $\widetilde{K}_{\mathcal{S}'}$  as  $\kappa_T^\epsilon$ . Since one  $\widetilde{g}_{\mathsf{K}_{S'}}$  corresponds to one  $\widetilde{K}_{\mathcal{S}'}$ ,  $|\mathcal{K}_T^\epsilon | = |\mathcal{G}_T^\epsilon | = \mathcal{N}(\mathcal{G}_T^S,\epsilon ,\| \cdot \| _1)$ .
Based on the above, we can write the Rademacher complexity of  $\mathcal{G}_T$  as
$$
\hat {\mathcal {R}} _ {\mathcal {S}} \left(\mathcal {G} _ {T}\right) = \frac {1}{n} \underset {\boldsymbol {\sigma}} {\mathbb {E}} \left[ \sup  _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}} \in \mathcal {K} _ {T}} \sum_ {i = 1} ^ {n} \sigma_ {i} g _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} \left(\boldsymbol {z} _ {i}\right) \right].
$$
For any  $\lambda > 0$ , by Jensen's inequality,
$$
\begin{array}{l} e ^ {\lambda \hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {T})} \leq \mathbb {E} _ {\boldsymbol {\sigma}} \left[ e ^ {\lambda \left[ \frac {1}{n} \sup  _ {\mathcal {K} _ {S ^ {\prime}} \in \mathcal {K} _ {T}} \sum_ {i = 1} ^ {n} \sigma_ {i} g _ {\mathcal {K} _ {S ^ {\prime}}} (\boldsymbol {z} _ {i}) \right]} \right] \\ = \mathbb {E} _ {\boldsymbol {\sigma}} \left[ \sup  _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}} \in \mathcal {K} _ {T}} e ^ {\lambda \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} g _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} \left(\boldsymbol {z} _ {i}\right) \right]} \right]. \tag {8} \\ \end{array}
$$
Utilizing the definition of  $\epsilon$ -covering, the quantity on the exponent in Eq. (8) is
$$
\begin{array}{l} \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} g _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i}) \\ = \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} \left(g _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} \left(\boldsymbol {z} _ {i}\right) - \widetilde {g} _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} \left(\boldsymbol {z} _ {i}\right) + \widetilde {g} _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} \left(\boldsymbol {z} _ {i}\right)\right) \\ = \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} \widetilde {g} _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i}) + \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} \left(g _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i}) - \widetilde {g} _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i})\right) \\ \leq \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} \widetilde {g} _ {\mathsf {K} _ {S ^ {\prime}}} (\boldsymbol {z} _ {i}) + \frac {1}{n} \sum_ {i = 1} ^ {n} \left| g _ {\mathsf {K} _ {S ^ {\prime}}} (\boldsymbol {z} _ {i}) - \widetilde {g} _ {\mathsf {K} _ {S ^ {\prime}}} (\boldsymbol {z} _ {i}) \right| \\ = \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} \widetilde {g} _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i}) + \frac {1}{n} \left\| g _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} (\mathbf {Z}) - \widetilde {g} _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} (\mathbf {Z}) \right\| _ {1} \\ \leq \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} \widetilde {g} _ {\mathrm {K} _ {S ^ {\prime}}} \left(\boldsymbol {z} _ {i}\right) + \frac {\epsilon}{n}. \quad \text {(b y} \\ \end{array}
$$
Substitute this inequality into Eq. (8). We have
$$
\begin{array}{l} e ^ {\lambda \hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {T})} \leq \underset {\boldsymbol {\sigma}} {\mathbb {E}} \left[ \sup  _ {\mathcal {K} _ {S ^ {\prime}} \in \mathcal {K} _ {T}} e ^ {\lambda \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} \widetilde {g} _ {\mathcal {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i}) + \frac {\epsilon}{n} \right]} \right] \\ = e ^ {\frac {\lambda \epsilon}{n}} \underset {\boldsymbol {\sigma}} {\mathbb {E}} \left[ \sup  _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}} \in \mathcal {K} _ {T}} e ^ {\lambda \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} g _ {\widetilde {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i}) \right]} \right] \quad \left(\widetilde {g} _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i}) = g _ {\widetilde {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i})\right) \\ \stackrel {i)} {=} e ^ {\frac {\lambda \epsilon}{n}} \underset {\boldsymbol {\sigma}} {\mathbb {E}} \left[ \max  _ {\widetilde {K} _ {S ^ {\prime}} \in \mathcal {K} _ {T} ^ {\epsilon}} e ^ {\lambda \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} g _ {\widetilde {K} _ {S ^ {\prime}}} (\boldsymbol {z} _ {i}) \right]} \right] \\ \leq e ^ {\frac {\lambda \epsilon}{n}} \sum_ {\widetilde {K} _ {\mathcal {S} ^ {\prime}} \in \mathcal {K} _ {T} ^ {\epsilon}} \mathbb {E} _ {\boldsymbol {\sigma}} \left[ e ^ {\lambda \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \sigma_ {i} g _ {\widetilde {K} _ {\mathcal {S} ^ {\prime}}} \left(\boldsymbol {z} _ {i}\right) \right]} \right] \tag {9} \\ \end{array}
$$
where  $(i)$  is because we only use the  $g_{\widetilde{K}_{\mathcal{S}^{\prime}}}$  instead of  $g_{\mathsf{K}_{\mathcal{S}^{\prime}}}$ , which corresponds to  $\widetilde{K}_{\mathcal{S}^{\prime}} \in \mathcal{K}_T^\epsilon$  instead of  $\mathsf{K}_{\mathcal{S}^{\prime}} \in \mathcal{K}_T$ , so it is equivalent to take the maximum over  $\widetilde{K}_{\mathcal{S}^{\prime}} \in \mathcal{K}_T^\epsilon$ . Denote  $\xi(\sigma_1, \ldots, \sigma_n) = \frac{1}{n} \sum_{i=1}^{n} \sigma_i g_{\widetilde{K}_{\mathcal{S}^{\prime}}}(\boldsymbol{z}_i)$ . Note  $g_{\widetilde{K}_{\mathcal{S}^{\prime}}}(\boldsymbol{z}_i) \in [0, 1]$  for  $i \in [n]$ . Then for all  $i \in [n]$ ,
$$
\begin{array}{l} \sup  _ {\sigma} \left| \xi \left(\sigma_ {1}, \dots , \sigma_ {i}, \dots , \sigma_ {n}\right) - \xi \left(\sigma_ {1}, \dots , - \sigma_ {i}, \dots , \sigma_ {n}\right) \right| \\ \leq \sup  _ {\boldsymbol {\sigma}} \left| \frac {2}{n} \sigma_ {i} g _ {\widetilde {K} _ {\mathcal {S} ^ {\prime}}} (\boldsymbol {z} _ {i}) \right| \\ \leq \frac {2}{n}. \\ \end{array}
$$
Thus  $\xi (\sigma_1,\dots ,\sigma_n)$  satisfies the bounded difference property. Let  $c_{i} = \frac{2}{n}$ . By [36, 10],  $\xi$  is a sub-Gaussian variable and satisfies
$$
\begin{array}{l} \mathbb {E} \left[ e ^ {\lambda \xi} \right] \leq e ^ {\frac {\lambda^ {2}}{8} \sum_ {i = 1} ^ {n} c _ {i} ^ {2}} e ^ {\lambda \mathbb {E} [ \xi ]} \\ = e ^ {\frac {\lambda^ {2}}{2 n}} e ^ {\lambda \mathbb {E} [ \xi ]}. \\ \end{array}
$$
Since  $g_{\widetilde{K}_{\mathcal{S}^{\prime}}}$  is a singleton function,  $\mathbb{E}\left[\xi\right] = \frac{1}{n}\mathbb{E}_{\pmb{\sigma}}\left[\sum_{i=1}^{n}\sigma_{i}g_{\widetilde{K}_{\mathcal{S}^{\prime}}}\left(\pmb{z}_{i}\right)\right] = 0$ . Thus
$$
\mathbb {E} \left[ e ^ {\lambda \xi} \right] \leq e ^ {\frac {\lambda^ {2}}{2 n}}.
$$
Take this into Eq. (9), in total, we have
$$
e ^ {\lambda \hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {T})} \leq e ^ {\frac {\lambda \epsilon}{n}} \sum_ {\tilde {K} _ {\mathcal {S} ^ {\prime}} \in \mathcal {K} _ {T} ^ {\epsilon}} e ^ {\frac {\lambda^ {2}}{2 n}} = e ^ {\frac {\lambda \epsilon}{n} + \frac {\lambda^ {2}}{2 n}} | \mathcal {K} _ {T} ^ {\epsilon} |.
$$
Take the logarithm on both sides of the equation,
$$
\lambda \hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {T}) \leq \frac {\lambda \epsilon}{n} + \frac {\lambda^ {2}}{2 n} + \ln | \mathcal {K} _ {T} ^ {\epsilon} |.
$$
Divide  $\lambda$  on both sides,
$$
\hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {T}) \leq \frac {\epsilon}{n} + \frac {\lambda}{2 n} + \frac {1}{\lambda} \ln | \mathcal {K} _ {T} ^ {\epsilon} |
$$
By taking  $\lambda = \sqrt{2n\ln|\mathcal{K}_T^\epsilon|}$ , we get
$$
\hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {T}) \leq \frac {\epsilon}{n} + \sqrt {\frac {2 \ln | \mathcal {K} _ {T} ^ {\epsilon} |}{n}}.
$$
Take the infimum over  $\epsilon > 0$  and note  $|\mathcal{K}_T^\epsilon| = |\mathcal{G}_T^\epsilon| = \mathcal{N}(\mathcal{G}_T^S, \epsilon, \| \cdot \|_1)$ . We get
$$
\hat {\mathcal {R}} _ {\mathcal {S}} \left(\mathcal {G} _ {T}\right) \leq \inf  _ {\epsilon > 0} \left(\frac {\epsilon}{n} + \sqrt {\frac {2 \ln \mathcal {N} \left(\mathcal {G} _ {T} ^ {\mathcal {S}} , \epsilon , \| \big \| _ {1}\right)}{n}}\right).
$$
C.3 A lower bound of  $\hat{\mathcal{R}}_S(\mathcal{G}_T^U)$
Here we give a lower bound of  $\hat{\mathcal{R}}_S(\mathcal{G}_T^U)$ . Similar lower bounds for a linear model were proved in [4, 6] without the supremum. Our lower bound matches the trace term in the upper bound  $U_{1}$ , which shows the upper bound  $U_{1}$  is relatively tight.
Theorem 7.
$$
\hat {\mathcal {R}} _ {\mathcal {S}} \left(\mathcal {G} _ {T} ^ {U}\right) \geq \frac {B}{\sqrt {2} n} \sup  _ {\mathsf {K} _ {\mathcal {S} ^ {\prime}} \in \mathcal {K} _ {T}} \sqrt {\operatorname {T r} \left(\mathsf {K} _ {\mathcal {S} ^ {\prime}} (\mathbf {Z} , \mathbf {Z})\right)}.
$$
Proof. Recall
$$
\mathcal {G} _ {T} ^ {U} = \left\{g (\boldsymbol {z}) = \left\langle \beta , \Phi_ {\mathcal {S} ^ {\prime}} (\boldsymbol {z}) \right\rangle + \ell \left(\boldsymbol {w} _ {0}, \boldsymbol {z}\right): \| \boldsymbol {\beta} \| \leq B, \mathrm {K} _ {\mathcal {S} ^ {\prime}} \in \mathcal {K} _ {T} \right\}.
$$
The Rademacher complexity of  $\mathcal{G}_T^U$  is
C.4 Proof of Corollary 1
Corollary 1. Fix  $B > 0$ . Let  $\hat{\mathcal{R}}_S^{gf}(\mathcal{G}_T) = \min (U_1,U_2)$  where  $U_{1}$  and  $U_{2}$  are defined in Theorem 3. For any  $\delta \in (0,1)$ , with probability at least  $1 - \delta$  over the draw of an i.i.d. sample set  $\mathcal{S} = \{z_i\}_{i = 1}^n$ , if  $\frac{1}{n^2}\sum_{i,j}\mathsf{K}_T(\pmb {z}_i,\pmb {z}_j;\mathcal{S})\leq B^2$ , the following holds for  $\ell (\pmb {w}_T,\pmb {z})$  that trained from  $\mathcal{S}$ ,
where  $\pmb{w}_T = A_T(S)$  is the output from the gradient flow (1) at time  $T$  by using  $S$  as input.
Proof. Apply Theorem 1 to  $\mathcal{G}_T$  and  $S$ , for all  $g \in \mathcal{G}_T$
$$
\underset {z} {\mathbb {E}} \left[ g (\boldsymbol {z}) \right] - \frac {1}{n} \sum_ {i = 1} ^ {n} g (\boldsymbol {z} _ {i}) \leq 2 \hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {T}) + 3 \sqrt {\frac {\log (2 / \delta)}{2 n}}.
$$
Since  $g(z) \in \mathcal{G}_T$  corresponds to  $\ell(A_T(S'), z)$  for all  $S' \in \mathrm{supp}(\mu^{\otimes n})$  that  $\frac{1}{n^2} \sum_{i,j} \mathsf{K}_T(z_i', z_j'; S') \leq B^2$ , it holds for all these feasible  $S' \in \mathrm{supp}(\mu^{\otimes n})$  that
$$
\begin{array}{l} \underset {z} {\mathbb {E}} \left[ \ell \left(A _ {T} \left(\mathcal {S} ^ {\prime}\right), \boldsymbol {z}\right) \right] - \frac {1}{n} \sum_ {i = 1} ^ {n} \ell \left(A _ {T} \left(\mathcal {S} ^ {\prime}\right), \boldsymbol {z} _ {i}\right) \\ = L _ {\mu} \left(A _ {T} \left(\mathcal {S} ^ {\prime}\right)\right) - L _ {S} \left(A _ {T} \left(\mathcal {S} ^ {\prime}\right)\right) \quad (\text {b y}) \\ \leq 2 \hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {T}) + 3 \sqrt {\frac {\log (2 / \delta)}{2 n}}. \\ \end{array}
$$
From Theorem 3, we have  $\hat{\mathcal{R}}_{\mathcal{S}}(\mathcal{G}_T) \leq \min(U_1, U_2)$ . Define  $\hat{\mathcal{R}}_{\mathcal{S}}^{gf}(\mathcal{G}_T) = \min(U_1, U_2)$  and take into above inequality,
$$
\begin{array}{l} L _ {\mu} \left(A _ {T} \left(\mathcal {S} ^ {\prime}\right)\right) - L _ {S} \left(A _ {T} \left(\mathcal {S} ^ {\prime}\right)\right) \leq 2 \hat {\mathcal {R}} _ {\mathcal {S}} \left(\mathcal {G} _ {T}\right) + 3 \sqrt {\frac {\log (2 / \delta)}{2 n}} \\ \leq 2 \hat {\mathcal {R}} _ {S} ^ {g f} (\mathcal {G} _ {T}) + 3 \sqrt {\frac {\log (2 / \delta)}{2 n}}. \\ \end{array}
$$
Since above holds for all  $S' \in \operatorname{supp}(\mu^{\otimes n})$  that  $\frac{1}{n^2} \sum_{i,j} \mathsf{K}_T(z_i', z_j'; S') \leq B^2$ , and  $S$  is also in  $\operatorname{supp}(\mu^{\otimes n})$ , then if  $\frac{1}{n^2} \sum_{i,j} \mathsf{K}_T(z_i, z_j; S) \leq B^2$ , we have
$$
L _ {\mu} (A _ {T} (\mathcal {S})) - L _ {S} (A _ {T} (\mathcal {S})) \leq 2 \hat {\mathcal {R}} _ {\mathcal {S}} ^ {g f} (\mathcal {G} _ {T}) + 3 \sqrt {\frac {\log (2 / \delta)}{2 n}}.
$$
D Complete Proofs for Stochastic Gradient Flow
D.1 Proof of Theorem 4
Proof. For each time interval  $[t, t + 1]$  and data batch  $S_{t}$ , Stochastic gradient flow can be treated as full-batch gradient flow. Applying Theorem 2 for each  $[t, t + 1]$  and data batch  $S_{t}$ , we have
$$
\ell \left(\boldsymbol {w} _ {t + 1}, \boldsymbol {z}\right) = \sum_ {i \in \mathcal {S} _ {t}} - \frac {1}{m} \mathrm {K} _ {t, t + 1} \left(\boldsymbol {z}, \boldsymbol {z} _ {i}; \mathcal {S}\right) + \ell \left(\boldsymbol {w} _ {t}, \boldsymbol {z}\right), \tag {10}
$$
$$
\mathsf {K} _ {t, t + 1} (z, z _ {i}; \mathcal {S}) = \int_ {t} ^ {t + 1} \bar {\mathsf {K}} (\boldsymbol {w} _ {t}; z, z _ {i}) d t.
$$
For time  $T\in \mathbb{N}$
$$
\begin{array}{l} \ell (\boldsymbol {w} _ {T}, \boldsymbol {z}) - \ell (\boldsymbol {w} _ {0}, \boldsymbol {z}) = \sum_ {t = 0} ^ {T - 1} \ell (\boldsymbol {w} _ {t + 1}, \boldsymbol {z}) - \ell (\boldsymbol {w} _ {t}, \boldsymbol {z}) \\ = \sum_ {t = 0} ^ {T - 1} \sum_ {i \in S _ {t}} - \frac {1}{m} K _ {t, t + 1} \left(\boldsymbol {z}, \boldsymbol {z} _ {i}; \mathcal {S}\right). \quad \text {(t a k e s i n E q . (1 0))} \\ \end{array}
$$
D.2 Proof of Theorem 5
$$
\hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {T}) \leq \sum_ {t = 0} ^ {T - 1} \frac {B _ {t}}{n} \sqrt {\sup  _ {\mathrm {K} (\cdot , ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T}} \operatorname {T r} \left(\mathrm {K} _ {t , t + 1} (\mathbf {Z} , \mathbf {Z}) ; \mathcal {S} ^ {\prime}\right) + \sum_ {i \neq j} \Delta_ {t} (z _ {i} , z _ {j})}.
$$
Proof. Recall,
$$
\mathcal {G} _ {T} = \{g (\boldsymbol {z}) = \sum_ {t = 0} ^ {T - 1} \sum_ {i \in \mathcal {S} _ {t}} - \frac {1}{m} \mathsf {K} _ {t, t + 1} (\boldsymbol {z}, \boldsymbol {z} _ {i} ^ {\prime}; \mathcal {S} ^ {\prime}) + \ell (\boldsymbol {w} _ {0}, \boldsymbol {z}): \mathsf {K} (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T} \}.
$$
$$
\mathcal {K} _ {T} = \{(\mathsf {K} _ {0, 1} (\cdot , \cdot ; \mathcal {S} ^ {\prime}), \dots , \mathsf {K} _ {T - 1, T} (\cdot , \cdot ; \mathcal {S} ^ {\prime})): \mathcal {S} ^ {\prime} \in \operatorname {s u p p} (\mu^ {\otimes n}), \frac {1}{m ^ {2}} \sum_ {i, j \in \mathcal {S} _ {t}} \mathsf {K} _ {t, t + 1} (z _ {i} ^ {\prime}, z _ {j} ^ {\prime}; \mathcal {S} ^ {\prime}) \leq B _ {t} ^ {2} \}.
$$
For  $t = 0,1,\dots ,T - 1$  , let
$$
\mathcal {G} _ {t} = \{g (z) = \sum_ {i \in \mathcal {S} _ {t}} - \frac {1}{m} \mathrm {K} _ {t, t + 1} (z, z _ {i} ^ {\prime}; \mathcal {S} ^ {\prime}): \mathrm {K} (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T} \},
$$
$$
\mathcal {G} _ {T} \subseteq \mathcal {G} _ {0} \oplus \mathcal {G} _ {1} \oplus \dots \oplus \mathcal {G} _ {T - 1} \oplus \left\{\ell (\boldsymbol {w} _ {0}, \boldsymbol {z}) \right\}.
$$
Since the set on the RHS involves combinations of kernels induced from distinct training set  $S'$ , it is a strictly larger set than the LHS. Apply Theorem 3 bound  $U_{1}$  for each  $\mathcal{G}_t$  on  $S$ ,
$$
\hat {\mathcal {R}} _ {\mathcal {S}} \left(\mathcal {G} _ {t}\right) \leq \frac {B _ {t}}{n} \left(\sup  _ {\mathsf {K} (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T}} \operatorname {T r} \left(\mathsf {K} _ {t, t + 1} (\mathbf {Z}, \mathbf {Z}); \mathcal {S} ^ {\prime}\right) + \sum_ {i \neq j} \Delta_ {t} \left(z _ {i}, z _ {j}\right)\right) ^ {\frac {1}{2}}, \tag {11}
$$
where  $\Delta_t(\pmb{z}_i, \pmb{z}_j) = \frac{1}{2}\left[\sup_{\mathsf{K}(\cdot, \cdot; \mathcal{S}') \in \mathcal{K}_T} \mathsf{K}_{t,t+1}(\pmb{z}_i, \pmb{z}_j; \mathcal{S}') - \inf_{\mathsf{K}(\cdot, \cdot; \mathcal{S}') \in \mathcal{K}_T} \mathsf{K}_{t,t+1}(\pmb{z}_i, \pmb{z}_j; \mathcal{S}')\right]$ .
By the monotonicity and linear combination of Rademacher complexity [38] and take in (11),
$$
\begin{array}{l} \hat {\mathcal {R}} _ {\mathcal {S}} \left(\mathcal {G} _ {T}\right) \leq \hat {\mathcal {R}} _ {\mathcal {S}} \left(\mathcal {G} _ {0} \oplus \mathcal {G} _ {1} \oplus \dots \oplus \mathcal {G} _ {T - 1} \oplus \left\{\ell \left(\boldsymbol {w} _ {0}, \boldsymbol {z}\right) \right\}\right) \\ = \sum_ {t = 0} ^ {T - 1} \hat {\mathcal {R}} _ {\mathcal {S}} (\mathcal {G} _ {t}) + \hat {\mathcal {R}} _ {\mathcal {S}} (\{\ell (\boldsymbol {w} _ {0}, \boldsymbol {z}) \}) \\ \leq \sum_ {t = 0} ^ {T - 1} \frac {B _ {t}}{n} \left(\sup  _ {\mathsf {K} (\cdot , \cdot ; \mathcal {S} ^ {\prime}) \in \mathcal {K} _ {T}} \operatorname {T r} \left(\mathsf {K} _ {t, t + 1} (\mathbf {Z}, \mathbf {Z}); \mathcal {S} ^ {\prime}\right) + \sum_ {i \neq j} \Delta_ {t} (z _ {i}, z _ {j})\right) ^ {\frac {1}{2}}. \\ \end{array}
$$
E Complete Proofs for Case Study
E.1 Generalization bounds for Infinite-width NNs
E.1.1 Proof of Eq. (4)