SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\left. \mathbb {E} _ {P _ {W | S} P _ {\hat {W} | S}} \left[ \left| \left(\hat {\mathcal {L}} (S ^ {\prime}, W) - \hat {\mathcal {L}} (S, W)\right) - \left(\hat {\mathcal {L}} (S ^ {\prime}, \hat {W}) - \hat {\mathcal {L}} (S, \hat {W})\right) \right| \right] \leqslant \epsilon / 2. \right. \tag {37}
$$
Proof. Consider a distribution  $P_{\hat{W} |S}$  that satisfies (37). Denote  $\lambda^{*} = \frac{2n - 1}{4}$  and
$$
\Delta (S, \mathbf {Q}) := \sqrt {\frac {\mathbb {E} _ {\hat {W} \sim P _ {\hat {W} | S}} \left[ \right. D _ {K L} \left( \right.P _ {\hat {Y} | X , \hat {W}} ^ {\otimes 2 n} (\hat {Y} ^ {n} , \hat {Y} ^ {\prime n} | X ^ {n} , X ^ {\prime n} , \hat {W}) \left\| \right. \mathbf {Q}\left. \right)\left. \right] + \log (\sqrt {8 n} / \delta)}{\lambda^ {*}}} + \epsilon .
$$
Furthermore, we use the shorthand notations  $\hat{p} \coloneqq P_{\hat{Y}|X,\hat{W}}^{\otimes 2n}(\hat{Y}^n,\hat{Y}'^n|X^n,X'^n,\hat{W})$ . Then,
$$
\mathbb {P} _ {S \sim \mu^ {\otimes n}} \left(\mathbb {E} _ {W \sim P _ {W | S}} [ \mathrm {g e n} (S, W) ] > \sqrt {\frac {\log (2 / \delta)}{2 n}} + \Delta (S, \mathbf {Q})\right)
$$
$$
\begin{array}{l} \stackrel {(a)} {\leqslant} \mathbb {P} _ {(S, S ^ {\prime}) \sim \mu^ {\otimes 2 n}} \left(\mathbb {E} _ {W \sim P _ {W | S}} \left[ \hat {\mathcal {L}} (S ^ {\prime}, W) - \hat {\mathcal {L}} (S, W) \right] > \Delta (S, \mathbf {Q})\right) + \delta / 2 \\ \stackrel {(b)} {\leqslant} \mathbb {P} _ {(S, S ^ {\prime}) \sim \mu^ {\otimes 2 n}} \left(\mathbb {E} _ {W \sim P _ {W | S}} \left[ \lambda^ {*} \left(\hat {\mathcal {L}} (S ^ {\prime}, W) - \hat {\mathcal {L}} (S, W)\right) ^ {2} \right] > \lambda^ {*} \Delta (S, \mathbf {Q}) ^ {2}\right) + \delta / 2 \\ \stackrel {(c)} {\leqslant} \mathbb {P} _ {(S, S ^ {\prime}) \sim \mu^ {\otimes 2 n}} \left(\mathbb {E} _ {\hat {W} \sim P _ {\hat {W} | S}} \left[ \lambda^ {*} \left(\hat {\mathcal {L}} (S ^ {\prime}, \hat {W}) - \hat {\mathcal {L}} (S, \hat {W})\right) ^ {2} \right] > \lambda^ {*} \Delta (S, \mathbf {Q}) ^ {2}\right) + \delta / 2 \\ = \mathbb {P} _ {(S, S ^ {\prime}) \sim \mu^ {\otimes 2 n}} \left(\lambda^ {*} \mathbb {E} _ {\hat {W}, \hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim P _ {\hat {W} | S} \hat {p}} \left[ \left(\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)\right) ^ {2} \right] \geqslant \lambda^ {*} \Delta (S, \mathbf {Q}) ^ {2}\right) \\ + \delta / 2 \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(d)} {\leqslant} \mathbb {P} _ {(S, S ^ {\prime}) \sim \mu^ {\otimes 2 n}} \left(D _ {K L} \left(P _ {\hat {W} | S} \hat {\rho} \| P _ {\hat {W} | S} \mathbf {Q}\right) \right. \tag {38} \\ \left. + \log \mathbb {E} _ {\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim \mathbf {Q}} \left[ \exp \left(\lambda^ {*} \left(\hat {\mathcal {L}} \left(S ^ {\prime}, Y ^ {\prime n}\right) - \hat {\mathcal {L}} (S, Y ^ {n})\right)\right) ^ {2} \right] \geqslant \lambda^ {*} \Delta (S, \mathbf {Q}) ^ {2}\right) \\ + \delta / 2 \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(e)} {\leqslant} \mathbb {P} _ {(S, S ^ {\prime}) \sim \mu^ {\otimes 2 n}} \left( \right.\log \mathbb {E} _ {\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim \mathbf {Q}} \left[ e ^ {\lambda^ {*} \left(\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)\right) ^ {2}} \right] \geqslant \\ \log \mathbb {E} _ {S, S ^ {\prime}, \hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim P _ {S} P _ {S ^ {\prime}}} \mathbf {Q} \left[ e ^ {\lambda^ {*} \left(\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)\right) ^ {2}} \right] + \log (2 / \delta) \\ + \delta / 2 \\ \end{array}
$$
$$
\stackrel {(f)} {\leqslant} \delta , \tag {39}
$$
where  $(a)$  holds by Hoeffding inequality and using the fact that for arbitrary random variables  $U, V$  and constants  $a, b \in \mathbb{R}$ ,  $\mathbb{P}(U + V > a + b) \leqslant \mathbb{P}(U > a) + \mathbb{P}(V > b)$ ,  $(b)$  by applying the Jensen inequality on the convex function  $f(x) = x^2$ ,  $(c)$  by using the distortion function (37) and since the loss is bounded by one,  $(d)$  by using the Donsker-Varadhan's variational representation lemma,  $(e)$  is shown in the following, and  $(f)$  by Markov inequality.
Hence, it remains to show the step  $(e)$ . To this end, it is sufficient upper bound
$$
\log \mathbb {E} _ {S, S ^ {\prime}, \hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim P _ {S} P _ {S ^ {\prime}} \mathbf {Q}} \left[ e ^ {\lambda^ {*} \left(\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)\right) ^ {2}} \right]
$$
by  $\log (\sqrt{2n})$ . Note that this terms equals
$$
\log \mathbb {E} _ {Y ^ {n}, Y ^ {\prime n}, \hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim P _ {Y} \otimes 2 n} \mathbf {Q} _ {1} \left[ e ^ {\lambda^ {*} \left(\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)\right) ^ {2}} \right],
$$
where  $\mathbf{Q}_1$  is equal to  $\mathbb{E}_{P_{X^n|Y^n}P_{X'm|Y'n}}\left[\mathbf{Q}(\hat{Y}^n, \hat{Y}_i'^m | X^n, Y^n, X'^n, Y'^n)\right]$ . Now,
$$
\mathbb {E} _ {Y ^ {n}, Y ^ {\prime n}, \hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim \mu_ {Y} ^ {\otimes 2 n}} \mathbf {Q _ {1}} (\hat {Y} ^ {n}, \hat {Y} _ {i} ^ {\prime n} | Y ^ {n}, Y ^ {\prime n}) \left[ \begin{array}{l} \lambda^ {*} \Bigg (\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right) \Bigg) ^ {2} \\ e \end{array} \right]
$$
$$
\begin{array}{l} \stackrel {(a)} {=} \mathbb {E} _ {\mathfrak {Y}} \mathfrak {Y} ^ {n \times 2}, \hat {\mathfrak {Y}} ^ {n \times 2}, K ^ {n} \sim P _ {Y} ^ {\otimes 2 n} \mathbf {Q} _ {1} (\hat {\mathfrak {Y}} ^ {n \times 2} | \mathfrak {Y} ^ {n \times 2}) \operatorname {U n i f} (1, 2) ^ {\otimes n} \left[ e ^ {\lambda^ {*} \left(\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{\mathfrak {Y} _ {K _ {i}, i} \neq \hat {\mathfrak {Y}} _ {K _ {i}, i} \}} - \mathbb {1} _ {\{\mathfrak {Y} _ {K _ {i} ^ {c}, i} \neq \hat {\mathfrak {Y}} _ {K _ {i} ^ {c}, i} \}}\right)\right) ^ {2}} \right] \\ = \mathbb {E} _ {\mathfrak {Y} ^ {n \times 2}, \hat {\mathfrak {Y}} ^ {n \times 2} \sim P _ {Y} ^ {\otimes 2 n}} \mathbf {Q} _ {\mathbf {1}} (\hat {\mathfrak {Y}} ^ {n \times 2} | \mathfrak {Y} ^ {n \times 2}) \mathbb {E} _ {K ^ {n}} \sim \mathrm {U n i f} (1, 2) ^ {\otimes n} \left[ e ^ {\lambda^ {*} \left(\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{\mathfrak {Y} _ {K _ {i}, i} \neq \hat {\mathfrak {Y}} _ {K _ {i}, i} \}} - \mathbb {1} _ {\{\mathfrak {Y} _ {K _ {i} ^ {c}, i} \neq \hat {\mathfrak {Y}} _ {K _ {i} ^ {c}, i} \}}\right)\right) ^ {2}} \right] \\ \end{array}
$$
$$
\stackrel {(b)} {\leqslant} \sqrt [ 6 ]{2 n}, \tag {40}
$$
where  $(a)$  is concluded by symmetry of  $\mathbf{Q}_1$  and  $(b)$  is deduced since
$$
\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{\mathfrak {Y} _ {K _ {i}, i} \neq \hat {\mathfrak {Y}} _ {K _ {i}, i} \}} - \mathbb {1} _ {\{\mathfrak {Y} _ {K _ {i} ^ {c}, i} \neq \hat {\mathfrak {Y}} _ {K _ {i} ^ {c}, i} \}}\right),
$$
is  $1 / \sqrt{n}$  -subgaussian process and hence
$$
\mathbb {E} _ {K ^ {n} \sim \mathrm {U n i f} (1, 2) ^ {\otimes n}} \left[ e ^ {\left(\frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{\mathfrak {Y} _ {K _ {i}, i} \neq \widehat {\mathfrak {Y}} _ {K _ {i}, i} \}} - \mathbb {1} _ {\{\mathfrak {Y} _ {K _ {i} ^ {c}, i} \neq \widehat {\mathfrak {Y}} _ {K _ {i} ^ {c}, i} \}}\right)\right) ^ {2} / (4 / (2 n - 1))} \right] \leqslant \sqrt {2 n},
$$
due to [Wai19, Theorem 2.6.IV]. This completes the proof.
E.7 Proof of Theorem 6
Let  $\mathbf{Q}$  be any fixed symmetric prior on  $(\hat{Y}^n,\hat{Y}^{\prime n})$  that could depend on  $(X^n,Y^n,X^{\prime n},Y^{\prime n})$ . Then, for any  $\delta \in \mathbb{R}^{+}$ , we show that with probability at least  $(1 - \delta)$  over choice of  $(S,S^{\prime},W)\sim P_{S^{\prime}}P_{S,W}$
$$
n h _ {D} \Big (\hat {\mathcal {L}} (S ^ {\prime}, W), \hat {\mathcal {L}} (S, W) \Big) \leqslant D _ {K L} \left(P _ {\hat {Y} | X, W} ^ {\otimes 2 n} (\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} | X ^ {n}, X ^ {\prime n}, W) \| \mathbf {Q}\right) + \log (n / \delta).
$$
Proof. The proof is a combination of proofs of Theorems 3 and 5. Denote the RHS of the bound in the theorem as  $\Delta(S, S', W)$ .
$$
\begin{array}{l} n h _ {D} \left(\hat {\mathcal {L}} \left(S ^ {\prime}, W\right), \hat {\mathcal {L}} (S, W)\right) \\ = n h _ {D} \left(\mathbb {E} _ {\hat {Y} ^ {\prime n} \sim P _ {\hat {Y} | X, W} ^ {\otimes n} (\hat {Y} ^ {\prime n} | X ^ {\prime n}, W)} \left[ \frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} \right], \mathbb {E} _ {\hat {Y} ^ {n} \sim P _ {\hat {Y} | X, W} ^ {\otimes n} (\hat {Y} ^ {n} | X ^ {n}, W)} \left[ \frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}} \right]\right) \\ \leqslant n \mathbb {E} _ {\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim P _ {\hat {Y} | X, W} ^ {\otimes 2 n} (\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} | X ^ {n}, X ^ {\prime n}, W)} \left[ h _ {D} \left(\frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}}, \frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right) \right] \\ \leqslant D _ {K L} \left(P _ {\hat {Y} | X, W} ^ {\otimes 2 n} \left(\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \mid X ^ {n}, X ^ {\prime n}, W\right) \| \mathbf {Q} \left(\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \mid X ^ {n}, Y ^ {n}, X ^ {\prime n}, Y ^ {\prime n}\right)\right) \\ + \log \mathbb {E} _ {\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim \mathbf {Q} (\hat {Y} ^ {n}, \hat {Y} _ {i} ^ {\prime n} | X ^ {n}, Y ^ {n}, X ^ {\prime n}, Y ^ {\prime n})} \left[ e ^ {n h _ {D} \left(\frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}}, \frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)} \right]. \tag {41} \\ \end{array}
$$
$$
\mathbb {P} _ {S, S ^ {\prime}, W} \left(n h _ {D} \left(\hat {\mathcal {L}} \left(S ^ {\prime}, W\right), \hat {\mathcal {L}} (S, W)\right) > \Delta \left(S, S ^ {\prime}, W\right)\right)
$$
$$
\begin{array}{l} \stackrel {(a)} {\leqslant} \mathbb {P} _ {Y ^ {n}, Y ^ {\prime n} \sim \mu^ {\otimes 2 n}} \left(\log \mathbb {E} _ {\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim \mathbf {Q}} \left[ e ^ {n h _ {D} \left(\frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}}, \frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)} \right] > \log (n / \delta)\right) \\ \stackrel {(b)} {\leqslant} \mathbb {P} _ {Y ^ {n}, Y ^ {\prime n} \sim \mu^ {\otimes 2 n}} \left(\log \mathbb {E} _ {\hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim \mathbf {Q}} \left[ e ^ {n h _ {D} \left(\frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}}, \frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)} \right] > \right. \\ \log \mathbb {E} _ {Y ^ {n}, Y ^ {\prime n}, \hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim \mu_ {Y} ^ {\otimes 2 n}} \mathbf {Q} \left[ e ^ {n h _ {D} \left(\frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}}, \frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)} \right] + \log (1 / \delta) \\ \end{array}
$$
$$
\begin{array}{c} (c) \\ \leqslant \delta , \end{array}
$$
where  $(a)$  follows by (41),  $(b)$  holds since
$$
\log \mathbb {E} _ {Y ^ {n}, Y ^ {\prime n}, \hat {Y} ^ {n}, \hat {Y} ^ {\prime n} \sim \mu_ {Y} ^ {\otimes 2 n}} \mathbf {Q} \left[ e ^ {n h _ {D} \left(\frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}}, \frac {1}{n} \sum_ {i \in [ n ]} \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)} \right] \leqslant \log (n),
$$
by the proof of Theorem 3, and  $(c)$  is derived using Markov inequality. This completes the proof.
E.8 Proof of Theorem 7
Let  $\mathbf{Q}$  be a type-III symmetric conditional prior over  $U^{2n}$  given  $X^{2n}, Y^{2n}$  and  $W_{e} \in \mathcal{W}_{e}$ , namely,  $\mathbf{Q}\Big((u_{\pi(1)}, \ldots, u_{\pi(2n)})|(x_1, \ldots, x_{2n}), (y_1, \ldots, y_{2n}), w\Big)$  remains the same for all permutations  $\pi \colon [2n] \mapsto [2n]$  that preserves the label, i.e.,  $y_{\pi(i)} = y_i$  for  $i \in [n]$ . Then, for any  $\lambda \in \mathbb{R}^+$ , we show that for  $K$ -classification learning task, with probability at least  $(1 - \delta)$  over  $(S, S', W) \sim P_{S'}P_{S,W}$ ,
$$
\hat {\mathcal {L}} (S ^ {\prime}, W) - \hat {\mathcal {L}} (S, W) \leqslant \frac {D _ {K L} \left(P _ {U | X , W _ {e}} ^ {\otimes 2 n} \left(U ^ {n} , U ^ {\prime n} | X ^ {n} , X ^ {\prime n} , W _ {e}\right) \| \mathbf {Q}\right) + (K + 2) / 2 + \log (1 / \delta)}{\lambda} + \frac {2 \lambda}{n}.
$$
The proof of the second part follows trivially using Hoeffding's inequality.
Proof. Note that whenever  $\lambda / n \geqslant 1.36$ , the bound trivially holds. Hence, assume that  $\lambda / n < 1.36$ . Denote the RHS of the bound of Part i. as  $\Delta(S, S', W)$ . We use the shorthand notation  $p_1 := P_{U|X, W_e}^{\otimes 2n}\left(U^n, U'^n | X^n, X'^n, W_e\right)$  and  $p_2 := P_{\hat{Y}|U, W_d}^{\otimes 2n}\left(\hat{Y}^n, \hat{Y}'^n | U^n, U'^n, W_d\right)$ .
Note that  $(S, S', W) \sim P_{S'} P_{S,W}$ . Now,
$$
\begin{array}{l} \mathbb {P} _ {S, S ^ {\prime}, W} \left(\hat {\mathcal {L}} \left(S ^ {\prime}, W\right) - \hat {\mathcal {L}} (S, W) > \Delta \left(S, S ^ {\prime}, W\right)\right) \\ = \mathbb {P} _ {S, S ^ {\prime}, W} \left(\lambda \mathbb {E} _ {p _ {1} p _ {2}} \left[ \frac {1}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right) \right] \geqslant \lambda \Delta (S, S ^ {\prime}, W)\right) \\ \leqslant^ {(a)} \mathbb {P} _ {S, S ^ {\prime}, W} \left(D _ {K L} \left(p _ {1} p _ {2} \| \mathbf {Q} p _ {2}\right) \right. \tag {42} \\ \end{array}
$$
$$
\left. + \log \mathbb {E} _ {\mathbf {Q} p _ {2}} \left[ \exp \left(\lambda \left(\hat {\mathcal {L}} \left(S ^ {\prime}, Y ^ {\prime n}\right) - \hat {\mathcal {L}} (S, Y ^ {n})\right)\right)\right) \right] \geqslant \lambda \Delta (S, S ^ {\prime}, W)
$$
$$
\begin{array}{l} \stackrel {(c)} {\leqslant} \mathbb {P} _ {S, S ^ {\prime}, W} \left( \right.\log \mathbb {E} _ {\mathbf {Q} p _ {2}} \left[ e ^ {\frac {\lambda}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)} \right] \geqslant \\ \log \mathbb {E} _ {P _ {S ^ {\prime}} P _ {S, W} \mathbf {Q} p _ {2}} \left[ e ^ {\frac {\lambda}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)} \right] + \log (1 / \delta) \\ \end{array}
$$
$$
\stackrel {(d)} {\leqslant} \delta , \tag {43}
$$
where  $(a)$  by using the Donsker-Varadhan's variational representation lemma,  $(b)$  since due to proof of Theorem 4 (in Appendix E.5),
$$
\log \mathbb {E} _ {P _ {S ^ {\prime}} P _ {S, W}} \mathbf {Q} p _ {2} \left[ e ^ {\frac {\lambda}{n} \sum_ {i \in [ n ]} \left(\mathbb {1} _ {\{Y _ {i} ^ {\prime} \neq \hat {Y} _ {i} ^ {\prime} \}} - \mathbb {1} _ {\{Y _ {i} \neq \hat {Y} _ {i} \}}\right)} \right] \leqslant \frac {2 \lambda^ {2}}{n} + \frac {K + 2}{2},
$$
when  $\lambda / n < 1.36$ , and  $(d)$  by Markov inequality. This completes the proof.
E.9 Proof of Lemma 3
Proof. The proof takes its main elements from [Dev83]. Let  $Y_{1}, Y_{2}, \ldots$  and  $Y_{1}', Y_{2}', \ldots$  be i.i.d. realizations of  $Y \sim p \in [K]$ . Denote  $P_{Y}(k) = p_{k}$  for  $k \in [K]$ . Let  $N$  and  $N'$  be two independent Poisson random variables with mean  $n$ . Let  $U_{i}$  and  $U_{i}'$ ,  $i \in [K]$ , be number of occurrences of value  $k$  among  $Y^{[n]}$  and  $Y^{[n']}$ , respectively. Moreover, let  $V_{i}$  and  $V_{i}'$ ,  $i \in [K]$ , be number of occurrences of value  $k$  among  $Y^{[N]}$  and  $Y^{[N']}$ , respectively. As shown in [Dev83],  $V_{1}, \ldots, V_{K}$  and  $V_{1}', \ldots, V_{K}'$  are independent Poisson random variables with means  $\mathbb{E}[V_{k}] = \mathbb{E}[V_{k}'] = np_{k}$ , for  $k \in [K]$ . In addition,  $(U_{1}, \ldots, U_{K})$  and  $(U_{1}', \ldots, U_{K}')$  are two independent multinomial  $(n, p_{1}, \ldots, p_{K})$  random vectors. Then,
$$
\begin{array}{l} \mathbb {E} \left[ e ^ {\lambda \| \hat {p} _ {Y ^ {n}} - \hat {p} _ {Y ^ {\prime n}} \| _ {1}} \right] \leqslant \mathbb {E} \left[ e ^ {\frac {\lambda}{n} \left(\sum_ {i = 1} ^ {K} | U _ {i} - V _ {i} - \left(U _ {i} ^ {\prime} - V _ {i} ^ {\prime}\right) |\right)} e ^ {\frac {\lambda}{n} \left(\sum_ {i = 1} ^ {K} | V _ {i} - V _ {i} ^ {\prime} |\right)} \right] \\ \stackrel {(a)} {\leqslant} \sqrt {\mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} \left(\sum_ {i = 1} ^ {K} \left| U _ {i} - V _ {i} - \left(U _ {i} ^ {\prime} - V _ {i} ^ {\prime}\right) \right|\right)} \right] \mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} \left(\sum_ {i = 1} ^ {K} \left| V _ {i} - V _ {i} ^ {\prime} \right|\right)} \right]} \\ \leqslant \sqrt {\mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} \left(\sum_ {i = 1} ^ {K} | U _ {i} - V _ {i} | + | U _ {i} ^ {\prime} - V _ {i} ^ {\prime} |\right)} \right] \mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} \left(\sum_ {i = 1} ^ {K} | V _ {i} - V _ {i} ^ {\prime} |\right)} \right]} \\ \leqslant \sqrt {\mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} \left(| N - n | + | N ^ {\prime} - n |\right)} \right] \mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} \left(\sum_ {i = 1} ^ {K} \left| V _ {i} - V _ {i} ^ {\prime} \right|\right)} \right]}, \tag {44} \\ \end{array}
$$
where  $(a)$  is due to Cauchy-Schwarz inequality.
Recall that for random variable  $X$  having the Poisson distribution with mean  $\eta$ , we have  $\mathbb{E}\left[e^{tX}\right] = e^{\eta (e^t -1)}$ . Now,
$$
\begin{array}{l} \mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} (| N - n | + | N ^ {\prime} - n |)} \right] = \mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} (| N - n |)} \right] \mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} (| N ^ {\prime} - n |)} \right] \\ \leqslant \mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} (N - n)} + e ^ {\frac {2 \lambda}{n} (n - N)} \right] \mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} \left(N ^ {\prime} - n\right)} + e ^ {\frac {2 \lambda}{n} \left(n - N ^ {\prime}\right)} \right] \\ \leqslant 4 e ^ {2 n \left(e ^ {\frac {2 \lambda}{n}} - 1 - \frac {2 \lambda}{n}\right)}, \tag {45} \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} \left(\sum_ {i = 1} ^ {K} \left| V _ {i} - V _ {i} ^ {\prime} \right|\right)} \right] \leqslant \prod_ {i = 1} ^ {K} \left(\mathbb {E} \left[ e ^ {\frac {2 \lambda}{n} \left(V _ {i} ^ {\prime} - V _ {i}\right)} + e ^ {\frac {2 \lambda}{n} \left(\hat {V} _ {i} ^ {\prime} - V _ {i} ^ {\prime}\right)} \right]\right) \\ = 2 ^ {K} \prod_ {i = 1} ^ {K} e ^ {n p _ {i} \left(e ^ {\frac {2 \lambda}{n}} - 1 + e ^ {- \frac {2 \lambda}{n}} - 1\right)} \\ = 2 ^ {K} e ^ {n \left(e ^ {\frac {2 \lambda}{n}} - 1 + e ^ {- \frac {2 \lambda}{n}} - 1\right)}, \tag {46} \\ \end{array}
$$
Combining (44),(45), and (45) yield
$$
\begin{array}{l} \log \mathbb {E} \left[ e ^ {\lambda \| \hat {p} _ {S} - \hat {p} _ {S ^ {\prime}} \| _ {1}} \right] \stackrel {(a)} {\leqslant} \frac {K + 2}{2} + n \left(e ^ {\frac {2 \lambda}{n}} - 1 - \frac {2 \lambda}{n} + \frac {1}{2} e ^ {\frac {2 \lambda}{n}} + \frac {1}{2} e ^ {- \frac {2 \lambda}{n}} - 1\right) \\ = \frac {K + 2}{2} + n \left(\frac {1}{1 . 2} \left(\frac {2 \lambda}{n}\right) ^ {2} + \frac {7}{1 2} \left(\frac {2 \lambda}{n}\right) ^ {2}\right) \\ \leqslant \frac {K + 2}{2} + \frac {6 \lambda^ {2}}{n}, \\ \end{array}
$$
where  $(a)$  holds due to two inequalities that can be verified numerically: i) for  $x < 1.36$ ,  $e^x < 1 + x + \frac{x^2}{1.2}$  and ii) for  $x < 1.3$ ,  $\frac{e^x + e^{-x}}{2} < 1 + \frac{7}{12} x^2$ .