SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathcal {E} (\hat {A} _ {m, \lambda} ^ {\mathrm {R R R}}) ^ {1 / 2} \leq a \lambda^ {1 / 2} + \sigma_ {r + 1} (\Phi_ {Y | X}) + \| (A _ {\lambda} ^ {\mathrm {R R R}} - \hat {A} _ {m, \lambda} ^ {\mathrm {R R R}}) \Phi_ {X} \| _ {\mathcal {B} (\mathcal {H})} \\ = a \lambda^ {1 / 2} + \sigma_ {r + 1} \left(\Phi_ {Y | X}\right) + \| \left(P _ {B} A _ {\lambda} - P _ {\tilde {B}} \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}}\right) \Phi_ {X} \| _ {\mathcal {B} (\mathcal {H})} \\ \leq a \lambda^ {1 / 2} + \sigma_ {r + 1} \left(\Phi_ {Y | X}\right) + \| \left(\left(P _ {B} - P _ {\tilde {B}}\right) A _ {\lambda} \Phi_ {X} \right\| _ {\mathcal {B} (\mathcal {H})} + \| P _ {\tilde {B}} \left(A _ {\lambda} - \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}}\right) \Phi_ {X} \| _ {\mathcal {B} (\mathcal {H})} \\ \leq a \lambda^ {1 / 2} + \sigma_ {r + 1} (\Phi_ {Y | X}) + K \frac {\| \tilde {B} \tilde {B} ^ {*} - B B ^ {*} \| _ {\mathcal {B} (\mathcal {H})}}{\sigma_ {r} ^ {2} (B) - \sigma_ {r + 1} ^ {2} (B)} + \| (A _ {\lambda} - \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}}) \Phi_ {X} \| _ {\mathcal {B} (\mathcal {H})} \\ \end{array}
$$
where the last inequality is due to  $\| A_{\lambda}\| \leq a$  and [30, Proposition 4].
Recalling Lemma E.1, we observe that
$$
\mathcal {E} (\hat {A} _ {m, \lambda} ^ {\mathrm {R R R}}) ^ {1 / 2} \leq \sigma_ {r + 1} (\Phi_ {Y | X}) + K \frac {\| \tilde {B} \tilde {B} ^ {*} - B B ^ {*} \|}{\sigma_ {r} ^ {2} (B) - \sigma_ {r + 1} ^ {2} (B)} + \underbrace {a \lambda^ {1 / 2} + \| (A _ {\lambda} - \hat {A} _ {m , \lambda} ^ {\mathrm {K R R}}) \Phi_ {X} \|} _ {\leq \mathrm {r . h . s . o f t h e b o u n d i n L e m m a E . 1}}
$$
Therefore, to prove Lemma 4.7 for the RRR estimator we just need to bound  $\| \tilde{B}\tilde{B}^{*} - BB^{*}\|_{\mathcal{B}(\mathcal{H})}$ . To that end, observe that, after some algebra, one obtains
$$
\tilde {B} \tilde {B} ^ {*} - B B ^ {*} = A _ {\lambda} (\tilde {C} _ {Y X} - C _ {Y X}) ^ {*} + (\tilde {C} _ {Y X} - C _ {Y X}) A _ {\lambda} ^ {*} - A _ {\lambda} (\tilde {C} _ {\lambda} - C _ {\lambda}) A _ {\lambda} ^ {*} + (\tilde {A} _ {\lambda} - A _ {\lambda}) \tilde {C} _ {\lambda} (\tilde {A} _ {\lambda} - A _ {\lambda}) ^ {*},
$$
$$
\begin{array}{l} \| \tilde {B} \tilde {B} ^ {*} - B B ^ {*} \| _ {\mathcal {B} (\mathcal {H})} \leq 2 a \| \tilde {C} _ {Y X} - C _ {Y X} \| _ {\mathcal {B} (\mathcal {H})} + a ^ {2} \| P _ {X} \hat {C} P _ {X} - C \| _ {\mathcal {B} (\mathcal {H})} \\ + \| C _ {\lambda} ^ {- 1 / 2} \tilde {C} _ {\lambda} C _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \| (\tilde {A} _ {\lambda} - A _ {\lambda}) C _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} ^ {2}, \\ \end{array}
$$
follows using that  $\| A_{\lambda}\|_{\mathcal{B}(\mathcal{H})}\leq a$
$$
\| \hat {C} _ {Y X} - C _ {Y X} \| _ {\mathcal {B} (\mathcal {H})} \leq \| P _ {Y} (\hat {C} _ {Y X} - C _ {Y X}) P _ {X} \| _ {\mathcal {B} (\mathcal {H})} + \| P _ {Y} ^ {\perp} C _ {Y X} P _ {X} \| _ {\mathcal {B} (\mathcal {H})} + \| C _ {Y X} P _ {X} ^ {\perp} \| _ {\mathcal {B} (\mathcal {H})},
$$
$$
\| \tilde {C} _ {Y X} - C _ {Y X} \| _ {\mathcal {B} (\mathcal {H})} \leq \| \hat {C} _ {Y X} - C _ {Y X} \| _ {\mathcal {B} (\mathcal {H})} + 2 a K \varepsilon_ {1},
$$
where  $\varepsilon_{1}:= \max \{\| P_{X}^{\perp}C^{1 / 2}\|_{\mathcal{B}(\mathcal{H})},\| P_{Y}^{\perp}C^{1 / 2}\|_{\mathcal{B}(\mathcal{H})}\}$ . Similarly, we obtain
$$
\left\| \tilde {C} _ {\lambda} - C _ {\lambda} \right\| _ {\mathcal {B} (\mathcal {H})} \leq \left\| \hat {C} - C \right\| _ {\mathcal {B} (\mathcal {H})} + 2 K \varepsilon_ {1}. \tag {43}
$$
But,  $\varepsilon_{1}$  can be bounded by Lemma J.6. Indeed, provided  $\lambda \in ]0,\| C\|_{\mathcal{B}(\mathcal{H})}]$ , it holds with probability  $1 - \delta'$
$$
\varepsilon_ {1} \leq \sqrt {3 \lambda}
$$
$$
\begin{array}{l} \| C _ {\lambda} ^ {- 1 / 2} \tilde {C} _ {\lambda} C _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \leq \| C _ {\lambda} ^ {- 1 / 2} P _ {X} \hat {C} _ {\lambda} P _ {X} C _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} + \lambda \| C _ {\lambda} ^ {- 1 / 2} P _ {X} ^ {\perp} C _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ \leq \theta_ {2} ^ {2} \| \hat {C} _ {\lambda} ^ {- 1 / 2} P _ {X} \hat {C} _ {\lambda} P _ {X} C _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} + 1 \\ \leq \theta_ {2} ^ {2} \| \hat {C} _ {\lambda} ^ {1 / 2} P _ {X} \hat {C} _ {\lambda} ^ {- 1} P _ {X} C _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} + 1 \\ \leq \theta_ {2} ^ {2} \| \hat {C} _ {\lambda} ^ {1 / 2} P _ {X} \left(P _ {X} \hat {C} _ {\lambda} P _ {X}\right) ^ {\dagger} P _ {X} C _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} + 1, \\ \end{array}
$$
$$
\left\| C _ {\lambda} ^ {- 1 / 2} \tilde {C} _ {\lambda} C _ {\lambda} ^ {- 1 / 2} \right\| _ {\mathcal {B} (\mathcal {H})} \leq \theta_ {2} ^ {2} + 1 \leq 2. 2 5, \tag {44}
$$
provided, as above, that  $n\lambda^{\tau} > 32c_{\tau}\beta$
Therefore, setting  $\varepsilon_0\coloneqq \max \{a\| \hat{C}_{YX} - C_{YX}\|_{\mathcal{B}(\mathcal{H})},a^2\| \hat{C} -C\|_{\mathcal{B}(\mathcal{H})}\}$ , for all  $i\in [m]$  we have
$$
\left| \sigma_ {i} ^ {2} (\tilde {B}) - \sigma_ {i} ^ {2} (B) \right| \leq \| \tilde {B} \tilde {B} ^ {*} - B B ^ {*} \| \leq 3 \varepsilon_ {0} + 6. 9 3 K a ^ {2} \lambda^ {1 / 2} + 2. 2 5 \varepsilon_ {2} ^ {2}, \tag {45}
$$
where  $\varepsilon_{2}:=\|(A_{\lambda}-\hat{A}_{m,\lambda}^{\mathrm{KRR}})C_{\lambda}^{1/2}\|$  is the variance of Nystrom KRR estimator, and conclude that
$$
\mathcal {E} (\hat {A} _ {m, \lambda} ^ {\mathrm {R R R}}) ^ {1 / 2} \leq \sigma_ {r + 1} (\Phi_ {Y | X}) + K \frac {3 \varepsilon_ {0} + 6 . 9 3 K a ^ {2} \lambda^ {1 / 2} + 2 . 2 5 \varepsilon_ {2} ^ {2}}{\sigma_ {r} ^ {2} (B) - \sigma_ {r + 1} ^ {2} (B)} + a \lambda^ {1 / 2} + \varepsilon_ {2}.
$$
Therefore, the proof of Lemma 4.7 for RRR estimator directly follows from the bound on  $a\lambda^{1 / 2} + \varepsilon_2$  given in the proof of Theorem 4.6, and the fact that, see e.g. [29],  $\varepsilon_0\lesssim n^{-1 / 2}\lesssim \lambda^{1 / 2}$
G Excess risk of the Nyström PCR estimator
Recalling Eq. (27), NyströmPCR estimator is of the form
$$
\hat {A} _ {m} ^ {\mathrm {P C R}} = P _ {Y} \hat {C} _ {Y X} \llbracket P _ {X} \hat {C} P _ {X} \rrbracket_ {r} ^ {\dagger} = \tilde {C} _ {Y X} \llbracket \tilde {C} _ {\lambda} \rrbracket_ {r} = \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}} P _ {\tilde {C} _ {\lambda}},
$$
for  $\lambda = 0$  and with  $P_{\tilde{C}_{\lambda}}$  being the orthogonal projector onto leading  $r$  eigenspace of  $\tilde{C}_{\lambda}$ . So, to prove Lemma 4.7 for PCR estimator, denote  $\hat{A}_{m,\lambda}^{\mathrm{PCR}} \coloneqq \hat{A}_{m,\lambda}^{\mathrm{KRR}} P_{\tilde{C}_{\lambda}}$  for  $\lambda \geq 0$ , and let us define the population version  $A_{\lambda}^{\mathrm{PCR}} = A_{\lambda} P_{C_{\lambda}}$ , where  $P_{C_{\lambda}}$  being the orthogonal projector onto leading  $r$  eigenspace of  $C_{\lambda}$ . As in the previous section we start with decomposition
$$
\begin{array}{l} \mathcal {E} \left(\hat {A} _ {m} ^ {\mathrm {P C R}}\right) ^ {1 / 2} = \left\| \Phi_ {Y | X} - A _ {\lambda} \Phi_ {X} \right\| _ {\mathcal {B} \left(L _ {\pi} ^ {2}, \mathcal {H}\right)} + \left\| \left(A _ {\lambda} - A _ {\lambda} ^ {\mathrm {P C R}}\right) \Phi_ {X} \right\| _ {\mathcal {B} \left(L _ {\pi} ^ {2}, \mathcal {H}\right)} + \\ \| \left(A _ {\lambda} ^ {\mathrm {P C R}} - \hat {A} _ {m, \lambda} ^ {\mathrm {P C R}}\right) \Phi_ {X} \| _ {\mathcal {B} \left(L _ {\pi} ^ {2}, \mathcal {H}\right)} + \| \left(\hat {A} _ {m, \lambda} ^ {\mathrm {P C R}} - \hat {A} _ {m} ^ {\mathrm {P C R}}\right) \Phi_ {X} \| _ {\mathcal {B} \left(L _ {\pi} ^ {2}, \mathcal {H}\right)}. \\ \end{array}
$$
The first and the second term are easily bounded by  $\| \Phi_{Y|X} - A_{\lambda}\Phi_{X}\|_{\mathcal{B}(L_{\pi}^{2},\mathcal{H})}\leq a\lambda^{1 / 2}$ , and
$$
\begin{array}{l} \| \left(A _ {\lambda} - A _ {\lambda} ^ {\mathrm {P C R}}\right) \Phi_ {X} \| _ {\mathcal {B} \left(L _ {\pi} ^ {2}, \mathcal {H}\right)} = \| A _ {\lambda} \left(I - P _ {C _ {\lambda}}\right) \Phi_ {X} \| _ {\mathcal {B} \left(L _ {\pi} ^ {2}, \mathcal {H}\right)} \\ \leq a \| \left(I - P _ {C _ {\lambda}}\right) C ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \leq a \sigma_ {r + 1} \left(\Phi_ {X}\right). \\ \end{array}
$$
For the third term, start by observing that
$$
\begin{array}{l} \left\| \left(A _ {\lambda} ^ {\mathrm {P C R}} - \hat {A} _ {m, \lambda} ^ {\mathrm {P C R}}\right) \Phi_ {X} \right\| _ {\mathcal {B} \left(L _ {n} ^ {2}, \mathcal {H}\right)} = \left\| \left(A _ {\lambda} P _ {C _ {\lambda}} - \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}} P _ {\tilde {C} _ {\lambda}}\right) C ^ {1 / 2} \right\| _ {\mathcal {B} (\mathcal {H})} \\ \leq \left\| A _ {\lambda} \left(P _ {C _ {\lambda}} - P _ {\tilde {C} _ {\lambda}}\right) C ^ {1 / 2} \right\| _ {\mathcal {B} (\mathcal {H})} + \left\| \left(A _ {\lambda} - \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}}\right) P _ {\tilde {C} _ {\lambda}} C ^ {1 / 2} \right\| _ {\mathcal {B} (\mathcal {H})} \\ \leq a K \left\| P _ {C _ {\lambda}} - P _ {\tilde {C} _ {\lambda}} \right\| _ {\mathcal {B} (\mathcal {H})} + \left\| \left(A _ {\lambda} - \tilde {A} _ {m, \lambda} ^ {\mathrm {K R R}}\right) P _ {\tilde {C} _ {\lambda}} C ^ {1 / 2} \right\| _ {\mathcal {B} (\mathcal {H})} \\ \leq a K \frac {\| \tilde {C} _ {\lambda} - C _ {\lambda} \| _ {\mathcal {B} (\mathcal {H})}}{\sigma_ {r} ^ {2} (\Phi_ {X}) - \sigma_ {r + 1} ^ {2} (\Phi_ {X})} + \| (A _ {\lambda} - \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}}) P _ {\tilde {C} _ {\lambda}} C ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ \leq K \frac {\varepsilon_ {0} + 2 a K \varepsilon_ {1}}{\sigma_ {r} ^ {2} (\Phi_ {X}) - \sigma_ {r + 1} ^ {2} (\Phi_ {X})} + \| (A _ {\lambda} - \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}}) P _ {\tilde {C} _ {\lambda}} C ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ \end{array}
$$
where the second last inequality is due to [30, Proposition 4] and the last one uses Eq. (43). Moreover, we have that
$$
\begin{array}{l} \| \left(A _ {\lambda} - \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}}\right) P _ {\tilde {C} _ {\lambda}} C ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \leq \| \left(A _ {\lambda} - \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}}\right) C _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \| C _ {\lambda} ^ {- 1 / 2} \tilde {C} _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \| \tilde {C} _ {\lambda} ^ {- 1 / 2} P _ {\tilde {C} _ {\lambda}} C ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ \leq \varepsilon_ {2} \sqrt {1 + \theta_ {2} ^ {2}} \| P _ {\tilde {C} _ {\lambda}} \tilde {C} _ {\lambda} ^ {- 1 / 2} C ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ \leq \varepsilon_ {2} \sqrt {1 + \theta_ {2} ^ {2}} \| C _ {\lambda} ^ {1 / 2} \tilde {C} _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})}, \\ \end{array}
$$
where we have used Eq. (44) and the fact that  $P_{\tilde{C}_{\lambda}}$  is the spectral projector of  $\tilde{C}_{\lambda}^{-1 / 2}$ . Therefore, due to
$$
\begin{array}{l} \| C _ {\lambda} ^ {1 / 2} \tilde {C} _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} = \| C _ {\lambda} ^ {1 / 2} [ P _ {X} ^ {\perp} + P _ {X} ] \tilde {C} _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \leq \| C _ {\lambda} ^ {1 / 2} P _ {X} \tilde {C} _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} + \| C _ {\lambda} ^ {1 / 2} P _ {X} ^ {\perp} \tilde {C} _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ \leq \theta_ {2} \| \hat {C} _ {\lambda} ^ {1 / 2} P _ {X} \tilde {C} _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} + \| C _ {\lambda} ^ {1 / 2} P _ {X} ^ {\perp} \| _ {\mathcal {B} (\mathcal {H})} \| \tilde {C} _ {\lambda} ^ {- 1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ \leq \theta_ {2} \| \hat {C} _ {\lambda} ^ {1 / 2} P _ {X} \left(P _ {X} \hat {C} _ {\lambda} P _ {X}\right) ^ {\dagger} P _ {X} \hat {C} _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} ^ {1 / 2} + \| C _ {\lambda} ^ {1 / 2} P _ {X} ^ {\perp} \| _ {\mathcal {B} (\mathcal {H})} \lambda^ {- 1 / 2} \\ \leq \theta_ {2} + \varepsilon_ {1} \lambda^ {- 1 / 2} \\ \end{array}
$$
$$
\| (A _ {\lambda} - \hat {A} _ {m, \lambda} ^ {\mathrm {K R R}}) P _ {\tilde {C} _ {\lambda}} C ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \leq \varepsilon_ {2} \left(1. 6 8 + 1. 5 \lambda^ {- 1 / 2} \varepsilon_ {1}\right),
$$
provided that  $n\lambda^{\tau} > 32c_{\tau}\beta$
Finally for the last term, observe that  $\llbracket \tilde{C}_{\lambda}\rrbracket_{r}^{\dagger}$  and  $\llbracket \tilde{C}_0\rrbracket_r^\dagger$  share the same eigenvectors, and hence  $\llbracket \tilde{C}_0\rrbracket_r^\dagger -\llbracket \tilde{C}_\lambda \rrbracket_r^\dagger = \lambda \llbracket \tilde{C}_\lambda \tilde{C}_0\rrbracket_r^\dagger$ . Hence, it holds that
$$
\begin{array}{l} \| (\hat {A} _ {\hat {m}, \lambda} ^ {\mathrm {P C R}} - \hat {A} _ {\hat {m}} ^ {\mathrm {P C R}}) \Phi_ {X} \| _ {\mathcal {B} (L _ {2} ^ {2}, \mathcal {H})} = \| \tilde {C} _ {Y X} ([ \tilde {C} _ {\tilde {\lambda}} ] _ {r} ^ {\dagger} - [ \tilde {C} _ {0} ] ] _ {r} ^ {\dagger}) C ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} = \lambda \| \tilde {C} _ {Y X} \tilde {C} _ {\lambda} ^ {- 1} [ \tilde {C} _ {0} ] ] _ {r} ^ {\dagger} C ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ \leq \lambda \| \tilde {C} _ {Y X} \tilde {C} _ {\lambda} ^ {- 1} [ [ \tilde {C} _ {0} ] ] _ {r} ^ {\dagger} \tilde {C} _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \| \tilde {C} _ {\lambda} ^ {- 1 / 2} C _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ = \lambda \| \tilde {C} _ {Y X} \tilde {C} _ {\lambda} ^ {- 1 / 2} [ \tilde {C} _ {0} ] ] _ {r} ^ {\dagger} \| _ {\mathcal {B} (\mathcal {H})} \| \tilde {C} _ {\lambda} ^ {- 1 / 2} C _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})} \\ = \lambda \| [ [ \tilde {C} _ {0} ] ] _ {r} ^ {\dagger} \| _ {\mathcal {B} (\mathcal {H})} \| \tilde {B} \| _ {\mathcal {B} (\mathcal {H})} \| \tilde {C} _ {\lambda} ^ {- 1 / 2} C _ {\lambda} ^ {1 / 2} \| _ {\mathcal {B} (\mathcal {H})}. \\ \end{array}
$$
Now, recalling Eq. (45), we can bound
$$
\begin{array}{l} \left\| \tilde {B} \right\| _ {\mathcal {B} (\mathcal {H})} ^ {2} \leq \left\| C _ {\lambda} ^ {- 1 / 2} C _ {Y X} ^ {*} C _ {Y X} C _ {\lambda} ^ {- 1 / 2} \right\| _ {\mathcal {B} (\mathcal {H})} + \left\| B B ^ {*} - \tilde {B} \tilde {B} ^ {*} \right\| _ {\mathcal {B} (\mathcal {H})} \\ \leq a ^ {2} K ^ {2} + 3 \varepsilon_ {0} + 6. 9 3 K a ^ {2} \lambda^ {1 / 2} + 2. 2 5 \varepsilon_ {2} ^ {2}, \\ \end{array}
$$
$$
\lambda^ {1 / 2} \| [   [ \tilde {C} _ {0} ]   ] _ {r} ^ {\dagger} \| _ {\mathcal {B} (\mathcal {H})} = \frac {\lambda^ {1 / 2}}{\lambda_ {r} (P _ {X} \hat {C} P _ {X})} \leq \frac {\lambda^ {1 / 2}}{\lambda_ {r} (C) - \| C _ {\lambda} - \tilde {C} _ {\lambda} \| _ {\mathcal {B} (\mathcal {H})}} \leq \frac {\lambda^ {1 / 2}}{\sigma_ {r} ^ {2} (\Phi_ {X}) - \| \hat {C} - C \| _ {\mathcal {B} (\mathcal {H})} - 2 K \varepsilon_ {1}}.
$$
Thus, consequently, we obtain
$$
\begin{array}{l} \| (\hat {A} _ {m, \lambda} ^ {\mathrm {P C R}} - \hat {A} _ {m} ^ {\mathrm {P C R}}) \Phi_ {X} \| _ {\mathcal {B} \left(L _ {\pi} ^ {2}, \mathcal {H}\right)} \leq \lambda^ {1 / 2} \left(\theta_ {2} + \varepsilon_ {1} \lambda^ {- 1 / 2}\right) \left(a ^ {2} K ^ {2} + 3 \varepsilon_ {0} + 6. 9 3 K a ^ {2} \lambda^ {1 / 2} + 2. 2 5 \varepsilon_ {2} ^ {2}\right). \\ \frac {\lambda^ {1 / 2}}{\sigma_ {r} ^ {2} \left(\Phi_ {X}\right) - \left\| \hat {C} - C \right\| _ {\mathcal {B} (\mathcal {H})} - 2 K \varepsilon_ {1}}. \\ \end{array}
$$
To conclude, observe that  $r > n^{\frac{1}{\beta(1 + \beta)}}$  due to Assumption 4.4 implies that  $\sigma_{r + 1}(\Phi_X) \lesssim n^{-\frac{1}{2(1 + \beta)}}$ .
Therefore, collecting all the terms, under the assumptions of Lemma 4.7 we obtain
$$
\mathcal {E} (\hat {A} _ {m} ^ {\mathrm {P C R}}) ^ {1 / 2} \lesssim c _ {\mathrm {P C R}} n ^ {- \frac {1}{2 (1 + \beta)}},
$$
where  $c_{\mathrm{PCR}} = (\sigma_r^2(\Phi_X) - \sigma_{r+1}^2(\Phi_X))^{-1}$  is the problem dependant constant.
H Auxiliary results
Lemma H.1 ([30, Proposition 2] with  $\alpha = 1$ ): Under Assumption 4.2 it holds
$$
\left\| \Phi_ {X} \mathcal {K} _ {\pi} ^ {*} - C _ {Y X} C _ {\lambda} ^ {- 1} \Phi_ {X} \right\| \leq a \lambda^ {1 / 2}.
$$
Lemma H.2: Let  $A$  be a bounded operator. Under Assumptions 4.1 and 4.5, it holds
$$
\left\| C _ {Y X} A \right\| \leq a \| C A \| \tag {46}
$$
Proof of Lemma H.2: Note that under Assumption 4.5, as  $C_{XY}C_{YX} \preceq a^2 C^2$  it also holds  $A^* C_{XY}C_{YX}A \preceq a^2 AC^2 A$  and thus:
$$
\begin{array}{l} \left\| C _ {Y X} A \right\| = \left\| A ^ {*} C _ {Y X} C _ {Y X} A \right\| ^ {1 / 2} \\ \leq a \| A ^ {*} C ^ {2} A \| ^ {1 / 2} \\ = a \| C A \|. \\ \end{array}
$$
The next lemma is a consequence of Assumption 4.3 and will be used in our concentration inequalities.
Lemma H.3: Under Assumption 4.3, it holds  $\pi$ -almost surely for any  $\nu$ :
$$
\left\| C _ {\lambda} ^ {- (1 - \nu) / 2} \phi (x) \right\| ^ {2} \leq c _ {\tau} \lambda^ {- [ \tau - \nu ] +} K ^ {2 [ \nu - \tau ] +}.
$$
The two following corollaries can be obtained picking  $\nu = 0$  and  $\nu = -1$ :
$$
\left\| C _ {\lambda} ^ {- 1 / 2} \phi (x) \right\| ^ {2} \leq \frac {c _ {\tau}}{\lambda^ {\tau}} \quad a n d \quad \left\| C _ {\lambda} ^ {- 1} \phi (x) \right\| ^ {2} \leq \frac {c _ {\tau}}{\lambda^ {\tau + 1}}.
$$
Proof of Lemma H.3: By [18, Theorem 9], it holds  $c_{\tau} \coloneqq \| k_{\pi}^{\tau}\|_{\infty}^{2} = \operatorname{ess}\sup_{x\sim \pi}\sum_{i\in I}\mu_{i}^{\tau}|e_{i}(x)|^{2}$  (where  $(e_i)$  is defined in Section A.3, and we recall that  $(\sqrt{\mu_i} e_i)_{i\in \mathbb{N}}$  is an orthonormal basis of  $\mathcal{H}$ . Denoting  $\mu_{i} \coloneqq \lambda_{i}(C)$ , it holds
$$
\begin{array}{l} \left\| C _ {\lambda} ^ {- (1 - \nu) / 2} \phi (x) \right\| ^ {2} = \left\| \left(\sum_ {i \in I} (\mu_ {i} + \lambda) ^ {- (1 - \nu) / 2} (\sqrt {\mu_ {i}} e _ {i}) \otimes (\sqrt {\mu_ {i}} e _ {i})\right) \phi (x) \right\| ^ {2} \\ = \left(\sum_ {i \in I} \mu_ {i} e _ {i} (x) ^ {2} \left(\mu_ {i} + \lambda\right) ^ {- 1 + \nu}\right) \\ = \sum_ {i \in I} \mu_ {i} ^ {1 - \tau} \left(\mu_ {i} + \lambda\right) ^ {- 1 + \nu} \mu_ {i} ^ {\tau} e _ {i} (x) ^ {2} \\ = \sum_ {i \in I} \left(\frac {\mu_ {i}}{\mu_ {i} + \lambda}\right) ^ {1 - \tau} (\mu_ {i} + \lambda) ^ {\nu - \tau} \mu_ {i} ^ {\tau} e _ {i} (x) ^ {2} \\ \leq \sum_ {i \in I} (\mu_ {i} + \lambda) ^ {- (\tau - \nu)} \mu_ {i} ^ {\tau} e _ {i} (x) ^ {2} \\ \leq c _ {\tau} \lambda^ {- [ \tau - \nu ] +} K ^ {2 [ \nu - \tau ] +}. \\ \end{array}
$$
where we used  $\sup |\mu_i|\leq K^2$
I Deterministic sketching results
Lemma I.1: Denoting  $R \coloneqq I - \hat{C}_{\lambda}^{1/2} g_{\mathrm{KRR}}(\hat{C}) \hat{C}_{\lambda}^{1/2}$ , it holds
$$
R \hat {C} _ {\lambda} ^ {1 / 2} = R \hat {C} _ {\lambda} ^ {1 / 2} P _ {X} ^ {\perp}.
$$
Proof of Lemma I.1: This is a direct consequence of the fact that  $g_{\mathrm{KRR}}(\hat{C}) \hat{C}_{\lambda} P_X = P_X$ :
$$
R \hat {C} _ {\lambda} ^ {1 / 2} P _ {X} = \hat {C} _ {\lambda} ^ {1 / 2} P _ {X} - \hat {C} _ {\lambda} ^ {1 / 2} g _ {\mathrm {K R R}} (\hat {C}) \hat {C} _ {\lambda} P _ {X} = 0.
$$
J Concentration results
J.1 Generic concentration lemmas
All our concentration results derive from two versions of the Bernstein inequality. We first state an inequality for sums of random variables in a Hilbert space based on [70, Theorem 3.3.4], which itself derives from a result of [48].
Lemma J.1: Let  $(A_{i})_{1\leq i\leq n}$  be i.i.d. copies of a random variable  $A$  in a separable Hilbert space  $(H,\| \cdot \|)$ . Assume  $\mathbf{E}A = \mu$  and  $\exists \sigma >0,\exists L > 0,\forall p\geq 2,\mathbf{E}\| A - \mu \| ^p\leq \frac{1}{2} p!\sigma^2 L^{p - 2}$ . Then for any  $\delta \in ]0,1[$  it holds:
$$
P \left[ \left\| \frac {1}{n} \sum_ {i = 1} ^ {n} A _ {i} - \mu \right\| \leq \frac {2 L \log (2 / \delta)}{n} + \sqrt {\frac {2 \sigma^ {2} \log (2 / \delta)}{n}} \right] \geq 1 - \delta \tag {47}
$$
The assumption on the moments holds in particular when  $\operatorname{ess}\sup \| A\| \leq L / 2$  and  $\mathbf{E}[\| A\| ^2 ]\leq \sigma^2$ .
Proof of Lemma J.1: See proof of [11, Lemma E.3] for a precise derivation based on [70, Theorem 3.3.4].  $\square$
We now state a version of the Bernstein concentration inequality for self-ajoint operators in operator norm, which is a restatement of [34, Lemma 24]. In the following, we denote  $\mathrm{r_{eff}}(A) \coloneqq \mathrm{tr}(A) / \| A\|$  the effective rank of a nonnegative definite operator  $A$ .
Lemma J.2 (Bernstein for self-ajoint operators acting on a Hilbert): Let  $H$  be a separable Hilbert space and  $A_{i}$  be i.i.d. copies of a random variable  $A$  taking values in the space of self-adjoint Hilbert-Schmidt operators on  $H$ . Assume  $\mathbf{E}A = 0$ ,  $\operatorname{ess}\sup \| A\|_{\mathrm{op}} \leq c$  for some  $c > 0$  (where  $\| \cdot \|_{\mathrm{op}}$  denotes the operator norm) and that there exists a positive semi-definite trace class operator  $V$  such that  $\mathbf{E}[A^2] \preccurlyeq V$ . Then for any  $\delta \in ]0,1[$  and  $n \geq 1$  it holds
$$
P \left[ \left\| \frac {1}{n} \sum_ {i = 1} ^ {n} A _ {i} \right\| _ {\text {o p}} \geq \frac {2 c \beta}{3 n} + \sqrt {\frac {2 \| V \| \beta}{n}} \right] \leq \delta \quad \text {w h e r e} \quad \beta = \log \left(\frac {4 \mathrm {r} _ {\text {e f f}} (V)}{\delta}\right) \tag {48}
$$
Proof of Lemma J.2: See [34, Appendix B.7, Lemma 24].
J.2 Applied concentration lemmas
Lemma J.3: Let Assumption 4.1 hold. Let  $\delta \in ]0,1[$ . Then for i.i.d. samples  $(x_{i},y_{i})_{1\leq i\leq n}$  and any  $\lambda \in ]0,\| C\|_{\mathcal{B}(\mathcal{H})}]$  it holds
$$
P \left[ \left\| C _ {\lambda} ^ {- 1 / 2} (\hat {C} - C) C _ {\lambda} ^ {- 1 / 2} \right\| _ {\mathcal {B} (\mathcal {H})} \geq \frac {4 c _ {\tau} \beta}{3 n \lambda^ {\tau}} + \sqrt {\frac {2 c _ {\tau} \beta}{n \lambda^ {\tau}}} \right] \leq \delta \quad w h e r e \quad \beta = \log \left(\frac {8 K ^ {2}}{\delta \lambda}\right) \tag {49}
$$
Proof of Lemma J.3: We apply Lemma J.2 on the random variables  $A_{i} = \xi (X_{i})\otimes \xi (X_{i}) - C_{\lambda}^{-1 / 2}CC_{\lambda}^{-1 / 2}$  where  $\xi (X_i)\coloneqq C_\lambda^{-1 / 2}\phi (X_i)$ . It holds
$$
\begin{array}{l} \operatorname {e s s} \sup  \| A _ {i} \| _ {\mathcal {B} (\mathcal {H})} \leq 2 \operatorname {e s s} \sup  \| \xi (X _ {i}) \| ^ {2} \\ \leq \frac {2 c _ {\tau}}{\lambda^ {\tau}}. \quad (\text {b y} \\ \mathbf {E} \left[ A _ {i} ^ {2} \right] = \mathbf {E} \left[ \| \xi (X _ {i}) \| ^ {2} \xi (X _ {i}) \xi (X _ {i}) ^ {*} \right] - \left(C _ {\lambda} ^ {- 1 / 2} C C _ {\lambda} ^ {- 1 / 2}\right) ^ {2} \\ \preccurlyeq \mathbf {E} [ \| \xi (X _ {i}) \| ^ {2} \xi (X _ {i}) \xi (X _ {i}) ^ {*} ] \\ \precneqq \frac {c _ {\tau}}{\lambda^ {\tau}} \mathbf {E} [ \xi (X _ {i}) \xi (X _ {i}) ^ {*} ] \\ = \frac {c _ {\tau}}{\lambda^ {\tau}} C C _ {\lambda} ^ {- 1} \\ \end{array}
$$
Thus applying Lemma J.2 with  $c = \frac{2c_{\tau}}{\lambda^{\tau}}$  and  $V = \frac{c_{\tau}}{\lambda^{\tau}} CC_{\lambda}^{-1}$ , we get
$$
P \left[ \left\| \frac {1}{n} \sum_ {i = 1} ^ {n} A _ {i} \right\| \geq \frac {4 c _ {\tau} \beta}{3 n \lambda^ {\tau}} + \sqrt {\frac {2 c _ {\tau} \beta}{\lambda^ {\tau} n}} \right] \leq \delta \quad \text {w h e r e} \quad \beta = \log \left(\frac {8 K ^ {2}}{\delta \lambda}\right) \tag {50}
$$
where we used the fact that  $\| CC_{\lambda}^{-1}\|_{\mathcal{B}(\mathcal{H})}\leq 1$  and controlled the effective rank using  $\mathrm{tr}(CC_{\lambda}^{-1})\leq K^2 /\lambda$  and  $\| CC_{\lambda}^{-1}\|_{\mathcal{B}(\mathcal{H})} = \| C\|_{\mathcal{B}(\mathcal{H})} / (\| C\|_{\mathcal{B}(\mathcal{H})} + 1)\geq 1 / 2$  because  $\lambda \leq \| C\|_{\mathcal{B}(\mathcal{H})}$  by assumption.
Lemma J.4: Let Assumptions 4.1 and 4.3 hold. Let  $\delta \in ]0,1[$ . Then for i.i.d. samples  $(x_{i},y_{i})_{1\leq i\leq n}$  we get
$$
P \left[ \| \left(C - \hat {C}\right) C _ {\lambda} ^ {- 1 / 2} \| _ {\mathrm {o p}} \leq \epsilon (\lambda , \delta) \right] \geq 1 - \delta \tag {51}
$$