SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\mathbf {M}) \\ = \left[ \begin{array}{c c} \mathbf {U} & \mathbf {U} _ {\perp} \end{array} \right] \left(\mathbf {H} (\boldsymbol {\sigma} ^ {(k)}, \varepsilon_ {k}) \circ \left(\left[ \begin{array}{c} \mathbf {U} ^ {*} \\ \mathbf {U} _ {\perp} ^ {*} \end{array} \right] \left[ \begin{array}{c c} \mathbf {U} & \mathbf {U} _ {\perp} \end{array} \right] \left[ \begin{array}{c c} \mathbf {M} _ {1} & \mathbf {M} _ {2} \\ \mathbf {M} _ {3} & 0 \end{array} \right] \left[ \begin{array}{c c} \mathbf {V} ^ {*} \\ \mathbf {V} _ {\perp} ^ {*} \end{array} \right] \left[ \begin{array}{c c} \mathbf {V} & \mathbf {V} _ {\perp} \end{array} \right]\right)\right) \left[ \begin{array}{c} \mathbf {V} ^ {*} \\ \mathbf {V} _ {\perp} ^ {*} \end{array} \right] \\ = \left[ \begin{array}{c c} \mathbf {U} & \mathbf {U} _ {\perp} \end{array} \right] \left(\mathbf {H} (\boldsymbol {\sigma} ^ {(k)}, \varepsilon_ {k}) \circ \left[ \begin{array}{c c} \mathbf {M} _ {1} & \mathbf {M} _ {2} \\ \mathbf {M} _ {3} & 0 \end{array} \right]\right) \left[ \begin{array}{c} \mathbf {V} ^ {*} \\ \mathbf {V} _ {\perp} ^ {*} \end{array} \right] \\ = \left[ \begin{array}{c c} \mathbf {U} & \mathbf {U} _ {\perp} \end{array} \right] \left[ \begin{array}{c c} \mathbf {H} _ {1} ^ {(k)} \circ \mathbf {M} _ {1} & \mathbf {H} _ {2} ^ {(k)} \circ \mathbf {M} _ {2} \\ \mathbf {H} _ {3} ^ {(k)} \circ \mathbf {M} _ {3} & 0 \end{array} \right] \left[ \begin{array}{c} \mathbf {V} ^ {*} \\ \mathbf {V} _ {\perp} ^ {*} \end{array} \right] \in T _ {\mathbf {U}, \mathbf {V}}. \\ \end{array}
$$
Similarly, if  $\mathbf{M} \in T_{\mathbf{U},\mathbf{V}}^{\perp}$ , there exists  $\mathbf{M}_4 \in \mathbb{R}^{(n_1 - r_k) \times (n_2 - r_k)}$  such that  $\mathbf{M} = \mathbf{U}_{\perp} \mathbf{M}_4 (\mathbf{V}_{\perp})^*$  and
$$
\begin{array}{l} W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\mathbf {M}) = \left[ \begin{array}{c c} \mathbf {U} & \mathbf {U} _ {\perp} \end{array} \right] \left(\mathbf {H} (\boldsymbol {\sigma} ^ {(k)}, \varepsilon_ {k}) \circ \left[ \begin{array}{c c} 0 & 0 \\ 0 & \mathbf {M} _ {4} \end{array} \right]\right) \left[ \begin{array}{c} \mathbf {V} ^ {*} \\ \mathbf {V} _ {\perp} ^ {*} \end{array} \right] \\ = \left[ \begin{array}{c c} \mathbf {U} & \mathbf {U} _ {\perp} \end{array} \right] \left[ \begin{array}{c c} 0 & 0 \\ 0 & \mathbf {M} _ {4} / \varepsilon_ {k} ^ {2} \end{array} \right] \left[ \begin{array}{c} \mathbf {V} ^ {*} \\ \mathbf {V} _ {\perp} ^ {*} \end{array} \right] \in T _ {\mathbf {U}, \mathbf {V}} ^ {\perp}. \\ \end{array}
$$
Proof of Lemma B.6: Let  $\Xi \in \mathbb{R}^{n_1\times n_2}$  be arbitrary. We start with some simple but technical observations. First note that by Lemma B.1, if  $T_{k} = T_{\tilde{\mathbf{U}},\tilde{\mathbf{V}},S}$ ,
$$
\begin{array}{l} \mathbb {P} _ {T _ {k}} ^ {\perp} \boldsymbol {\Xi} = (\mathbf {I d} - \mathbb {P} _ {T _ {k}}) (\boldsymbol {\Xi}) \\ = \boldsymbol {\Xi} - \left(\widetilde {\mathbf {U}} \widetilde {\mathbf {U}} ^ {*} \boldsymbol {\Xi} + \mathbb {P} _ {S} \boldsymbol {\Xi} \widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} - \widetilde {\mathbf {U}} \widetilde {\mathbf {U}} ^ {*} \boldsymbol {\Xi} \widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*}\right) \tag {45} \\ = (\mathbf {I d} - \mathbb {P} _ {\tilde {\mathbf {U}}, \tilde {\mathbf {V}}}) \boldsymbol {\Xi} + (\mathbf {I d} - \mathbb {P} _ {S}) \boldsymbol {\Xi} \widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} \\ = \mathbb {P} _ {\tilde {\mathbf {U}}, \tilde {\mathbf {V}}} ^ {\perp} \boldsymbol {\Xi} + \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \tilde {\mathbf {V}} \tilde {\mathbf {V}} ^ {*}, \\ \end{array}
$$
$$
\left\langle \mathbb {P} _ {\tilde {\mathbf {U}}, \tilde {\mathbf {V}}} ^ {\perp} \boldsymbol {\Xi}, \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \tilde {\mathbf {V}} \tilde {\mathbf {V}} ^ {*} \right\rangle = \left\langle \tilde {\mathbf {U}} _ {\perp} \tilde {\mathbf {U}} _ {\perp} ^ {*} \boldsymbol {\Xi} \tilde {\mathbf {V}} _ {\perp} \tilde {\mathbf {V}} _ {\perp} ^ {*}, \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \tilde {\mathbf {V}} \tilde {\mathbf {V}} ^ {*} \right\rangle = 0, \tag {46}
$$
where we used that  $\mathbb{P}_{\widetilde{\mathbf{U}},\widetilde{\mathbf{V}}}^{\perp}\Xi = (\mathbf{Id} - \mathbb{P}_{\widetilde{\mathbf{U}},\widetilde{\mathbf{V}}})\Xi = \widetilde{\mathbf{U}}_{\perp}\widetilde{\mathbf{U}}_{\perp}^{*}\Xi \widetilde{\mathbf{V}}_{\perp}\widetilde{\mathbf{V}}_{\perp}^{*}$ , for  $\widetilde{\mathbf{U}}_{\perp}\in \mathbb{R}^{n_1\times (n_1 - r)}$  and  $\widetilde{\mathbf{V}}_{\perp}\in \mathbb{R}^{n_2\times (n_2 - r)}$  being the complementary orthonormal bases of  $\widetilde{\mathbf{U}}$  and  $\widetilde{\mathbf{V}}$ , and that  $\widetilde{\mathbf{V}}_{\perp}^{*}\widetilde{\mathbf{V}} = \mathbf{0}$ .
Second, let now  $\mathbf{U} \in \mathbb{R}^{n_1 \times r}$  and  $\mathbf{V} \in \mathbb{R}^{n_2 \times r}$  be matrices with  $r$  leading left and right singular vectors of  $\mathbf{X}^{(k)}$  in their columns which coincide with the matrices  $\mathbf{U}$  and  $\mathbf{V}$  from Definition 2.1 in their first  $r$  columns. Then it follows from (45) and (46) that
$$
\begin{array}{l} \left\| \mathbb {P} _ {T _ {k} ^ {\perp}} (\boldsymbol {\Xi}) \right\| _ {F} ^ {2} \\ = \left\| \mathbb {P} _ {\tilde {\mathbf {U}}, \tilde {\mathbf {V}}} ^ {\perp} \boldsymbol {\Xi} \right\| _ {F} ^ {2} + \left\| \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} \right\| _ {F} ^ {2} \\ = \left\| \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi} + \left(\mathbb {P} _ {\mathbf {U}, \widetilde {\mathbf {V}}} ^ {\perp} - \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp}\right) \boldsymbol {\Xi} \right\| _ {F} ^ {2} + \left\| \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \left(\mathbf {V V} ^ {*} + \left(\widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} - \mathbf {V V} ^ {*}\right)\right) \right\| _ {F} ^ {2} \tag {47} \\ \leq 2 \left(\left\| \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi} \right\| _ {F} ^ {2} + \left\| (\mathbb {P} _ {\mathbf {U}, \tilde {\mathbf {V}}} ^ {\perp} - \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp}) \boldsymbol {\Xi} \right\| _ {F} ^ {2} + \| \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \mathbf {V} \mathbf {V} ^ {*} \| _ {F} ^ {2} + \left\| \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} (\widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} - \mathbf {V} \mathbf {V} ^ {*}) \right\| _ {F} ^ {2}\right). \\ \end{array}
$$
By an argument analogous to (46), we observe that
$$
\left\| \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi} \right\| _ {F} ^ {2} + \| \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \mathbf {V} \mathbf {V} ^ {*} \| _ {F} ^ {2} = \langle \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi} + \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \mathbf {V} \mathbf {V} ^ {*}, \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi} + \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \mathbf {V} \mathbf {V} ^ {*} \rangle = \langle \widetilde {\boldsymbol {\Xi}}, \widetilde {\boldsymbol {\Xi}} \rangle ,
$$
where  $\widetilde{\Xi} = \mathbb{P}_{\mathcal{M}}(\Xi)$  is an element of the subspace  $\mathcal{M} = \mathcal{M}_1 \oplus \mathcal{M}_2 \subset \mathbb{R}^{n_1 \times n_2}$  that is the direct sum of the subspaces  $\mathcal{M}_1 := \{\mathbb{P}_{\mathbf{U},\mathbf{V}}^\perp \mathbf{Z} : \mathbf{Z} \in \mathbb{R}^{n_1 \times n_2}\}$  and  $\mathcal{M}_2 := \{\mathbb{P}_{S^c} \mathbf{Z} \mathbf{V} \mathbf{V}^* : \mathbf{Z} \in \mathbb{R}^{n_1 \times n_2}\}$ .
Let now  $W_{\mathbf{X}^{(k)}, \varepsilon_k}^{lr}$  be the rank promoting part of the weight operator from (9) and  $\mathbf{W}_{\mathbf{X}^{(k)}, \delta_k}^{sp}$  be the row-sparsity promoting part from (10). Note that the restriction of
$$
\bar {W} := \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} + \mathbb {P} _ {S ^ {c}} \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \mathbb {P} _ {S ^ {c}}
$$
to  $\mathcal{M}$  is invertible as its first summand is invertible on  $\mathcal{M}_1 = T_{\mathbf{U},\mathbf{V}}^\perp$ , its second summand is invertible on  $\mathcal{M}_2$  (recall that the weight operators are positive definite), and  $\mathcal{M}_1 \perp \mathcal{M}_2$ . Therefore it holds that
$$
\begin{array}{l} \left\| \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi} \right\| _ {F} ^ {2} + \| \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \mathbf {V} \mathbf {V} ^ {*} \| _ {F} ^ {2} \\ = \langle \widetilde {\Xi}, \widetilde {\Xi} \rangle = \left\langle \bar {W} _ {| \mathcal {M}} ^ {1 / 2} \widetilde {\Xi}, \bar {W} _ {| \mathcal {M}} ^ {- 1} \bar {W} _ {| \mathcal {M}} ^ {1 / 2} \widetilde {\Xi} \right\rangle \\ \leq \sigma_ {1} \left(\bar {W} _ {| \mathcal {M}} ^ {- 1}\right) \left\langle \widetilde {\Xi}, \bar {W} _ {| \mathcal {M}} \widetilde {\Xi} \right\rangle = \frac {1}{\sigma_ {\min } \left(\bar {W} _ {| \mathcal {M}}\right)} \left\langle \widetilde {\Xi}, \bar {W} \widetilde {\Xi} \right\rangle \\ \leq \frac {1}{\sigma_ {\min } \left(\left(\mathbb {P} _ {\mathbf {U} , \mathbf {V}} ^ {\perp} W _ {\mathbf {X} ^ {(k)} , \varepsilon_ {k}} ^ {l r} \mathbb {P} _ {\mathbf {U} , \mathbf {V}} ^ {\perp}\right) _ {| \mathcal {M}}\right) + \sigma_ {\min } \left(\left(\mathbb {P} _ {S ^ {c}} \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \mathbb {P} _ {S ^ {c}}\right) _ {| \mathcal {M}}\right)} \left\langle \widetilde {\Xi}, \bar {W} \widetilde {\Xi} \right\rangle \\ \leq \frac {1}{\frac {\varepsilon_ {k} ^ {2}}{\sigma_ {r + 1} ^ {2} (\mathbf {X} ^ {(k)})} + \frac {\delta_ {k} ^ {2}}{\rho_ {s + 1} ^ {2} (\mathbf {X} ^ {(k)})}} \Big \langle \boldsymbol {\Xi}, \bar {W} \boldsymbol {\Xi} \Big \rangle . \\ \end{array}
$$
In the first inequality, we used that  $\bar{W}_{|\mathcal{M}}$  is positive definite. In the second inequality, we used that  $\sigma_{\min}(A + B) \geq \sigma_{\min}(A) + \sigma_{\min}(B)$ , for any positive semidefinite operators  $A$  and  $B$ , and in the third inequality that  $\langle \widetilde{\Xi}, \bar{W} \widetilde{\Xi} \rangle \leq \langle \Xi, \bar{W} \Xi \rangle$ . The latter observation can be deduced as follows: Note that, by the self-adjointness of  $\bar{W}$ ,
$$
\begin{array}{l} \langle \Xi , \bar {W} \Xi \rangle = \langle \mathbb {P} _ {\mathcal {M}} (\Xi), \bar {W} \mathbb {P} _ {\mathcal {M}} (\Xi) \rangle + \langle \mathbb {P} _ {\mathcal {M}} (\Xi), \bar {W} \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\Xi) \rangle + \langle \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\Xi), \bar {W} \mathbb {P} _ {\mathcal {M}} (\Xi) \rangle + \langle \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\Xi), \bar {W} \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\Xi) \rangle \\ = \langle \widetilde {\Xi}, \bar {W} \widetilde {\Xi} \rangle + 2 \langle \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\Xi), \bar {W} \mathbb {P} _ {\mathcal {M}} (\Xi) \rangle + \langle \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\Xi), \bar {W} \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\Xi) \rangle . \\ \end{array}
$$
Since  $\bar{W}$  is positive semi-definite (due to the fact that both  $W_{\mathbf{X}^{(k)},\varepsilon_k}^{lr}$  and  $\mathbf{W}_{\mathbf{X}^{(k)},\delta_k}^{sp}$  are positive definite), all that remains is to argue that the mixed term on the right-hand side vanishes. To this end, note that  $\mathbb{P}_{\mathbf{U},\mathbf{V}}^{\perp}\mathbf{Z} = \mathbb{P}_{S^{\epsilon}}\mathbf{Z}\mathbf{V}\mathbf{V}^{*} = 0$ , for any  $\mathbf{Z}\in \mathcal{M}_{\perp}$  and compute
$$
\begin{array}{l} \langle \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\boldsymbol {\Xi}), \bar {W} \mathbb {P} _ {\mathcal {M}} (\boldsymbol {\Xi}) \rangle = \langle \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\boldsymbol {\Xi}), \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} (W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} (\mathbb {P} _ {\mathcal {M} _ {1}} (\boldsymbol {\Xi})))) \rangle + \langle \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\boldsymbol {\Xi}), \mathbb {P} _ {S ^ {c}} \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \mathbb {P} _ {S ^ {c}} \mathbb {P} _ {\mathcal {M} _ {2}} (\boldsymbol {\Xi}) \rangle \\ = \langle \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} (\mathbb {P} _ {\mathcal {M}} ^ {\perp} (\boldsymbol {\Xi})), W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} (\mathbb {P} _ {\mathcal {M} _ {1}} (\boldsymbol {\Xi}))) \rangle + \langle \mathbb {P} _ {S ^ {c}} \mathbb {P} _ {\mathcal {M}} ^ {\perp} (\boldsymbol {\Xi}) \mathbf {V V} ^ {*}, \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \rangle \\ = 0. \\ \end{array}
$$
We can now continue by estimating
$$
\begin{array}{l} \left\langle \boldsymbol {\Xi}, \bar {W} \boldsymbol {\Xi} \right\rangle = \left\langle \boldsymbol {\Xi}, \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi} \right\rangle + \left\langle \boldsymbol {\Xi}, \mathbb {P} _ {S ^ {c}} \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \right\rangle \\ = \left\langle \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} (\boldsymbol {\Xi}), W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} (\boldsymbol {\Xi}) \right\rangle + \left\langle \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi}, \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \right\rangle \tag {48} \\ \leq \left\langle \boldsymbol {\Xi}, W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} \boldsymbol {\Xi} \right\rangle + \left\langle \boldsymbol {\Xi}, \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \boldsymbol {\Xi} \right\rangle \\ = \left\langle \boldsymbol {\Xi}, W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}, \delta_ {k}} \boldsymbol {\Xi} \right\rangle , \\ \end{array}
$$
using the positive semidefiniteness of  $W_{\mathbf{X}^{(k)},\varepsilon_k}^{lr}$  and  $\mathbf{W}_{\mathbf{X}^{(k)},\delta_k}^{sp}$  in the last inequality. To be precise, the last inequality can be argued as follows: Due to complimentary supports  $S$  and  $S^c$ , we see that
$$
\begin{array}{l} \langle \boldsymbol {\Xi}, \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \boldsymbol {\Xi} \rangle = \langle \mathbb {P} _ {S} \boldsymbol {\Xi}, \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \mathbb {P} _ {S} \boldsymbol {\Xi} \rangle + \langle \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi}, \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \rangle \\ + \underbrace {\langle \mathbb {P} _ {S} \boldsymbol {\Xi} , \mathbf {W} _ {\mathbf {X} ^ {(k)} , \delta_ {k}} ^ {s p} \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \rangle} _ {= 0} + \underbrace {\langle \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} , \mathbf {W} _ {\mathbf {X} ^ {(k)} , \delta_ {k}} ^ {s p} \mathbb {P} _ {S} \boldsymbol {\Xi} \rangle} _ {= 0} \tag {49} \\ \geq \left\langle \mathbb {P} _ {S ^ {\infty}} \boldsymbol {\Xi}, \mathbf {W} _ {\mathbf {X} ^ {(k)}, \delta_ {k}} ^ {s p} \right. \mathbb {P} _ {S ^ {\infty}} \boldsymbol {\Xi} \rangle . \\ \end{array}
$$
Similarly, we note that  $W_{\mathbf{X}^{(k)}, \varepsilon_k}^{lr}$  acts diagonally on  $T_{\mathbf{U}, \mathbf{V}}$  and  $T_{\mathbf{U}, \mathbf{V}}^{\perp}$ . Indeed, we have by Lemma C.2 that if  $\mathbf{M} \in T_{\mathbf{U}, \mathbf{V}}$ , then  $W_{\mathbf{X}^{(k)}, \varepsilon_k}^{lr}(\mathbf{M}) \in T_{\mathbf{U}, \mathbf{V}}$  and if  $\mathbf{M} \in T_{\mathbf{U}, \mathbf{V}}^{\perp}$ , then  $W_{\mathbf{X}^{(k)}, \varepsilon_k}^{lr}(\mathbf{M}) \in T_{\mathbf{U}, \mathbf{V}}^{\perp}$ , which implies
$$
\langle \mathbb {P} _ {\mathbf {U}, \mathbf {V}} \boldsymbol {\Xi}, W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi}) \rangle = 0 \quad \mathrm {a n d} \quad \langle \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi}, W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\mathbb {P} _ {\mathbf {U}, \mathbf {V}} \boldsymbol {\Xi}) \rangle = 0
$$
due to the orthogonality of elements in  $T_{\mathbf{U},\mathbf{V}}^{\perp}$  and  $T_{\mathbf{U},\mathbf{V}}$ , respectively, and therefore it follows from  $\Xi = T_{\mathbf{U},\mathbf{V}}(\Xi) + T_{\mathbf{U},\mathbf{V}}^{\perp}(\Xi)$  that
$$
\begin{array}{l} \langle \boldsymbol {\Xi}, W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\boldsymbol {\Xi}) \rangle = \langle \mathbb {P} _ {\mathbf {U}, \mathbf {V}} \boldsymbol {\Xi}, W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\mathbb {P} _ {\mathbf {U}, \mathbf {V}} \boldsymbol {\Xi}) \rangle + \langle \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi}, W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi}) \rangle \\ + \underbrace {\left\langle \mathbb {P} _ {\mathbf {U} , \mathbf {V}} \boldsymbol {\Xi} , W _ {\mathbf {X} ^ {(k)} , \varepsilon_ {k}} ^ {I r} \left(\mathbb {P} _ {\mathbf {U} , \mathbf {V}} ^ {\perp} \boldsymbol {\Xi}\right) \right\rangle} _ {= 0} + \underbrace {\left\langle \mathbb {P} _ {\mathbf {U} , \mathbf {V}} ^ {\perp} \boldsymbol {\Xi} , W _ {\mathbf {X} ^ {(k)} , \varepsilon_ {k}} ^ {I r} \left(\mathbb {P} _ {\mathbf {U} , \mathbf {V}} \boldsymbol {\Xi}\right) \right\rangle} _ {= 0} \tag {50} \\ \geq \langle \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi}, W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}} ^ {l r} (\mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp} \boldsymbol {\Xi}) \rangle . \\ \end{array}
$$
Combining the previous estimates with (47) and noticing that
$$
\frac {1}{\frac {\varepsilon_ {k} ^ {2}}{\sigma_ {r + 1} ^ {2} (\mathbf {X} ^ {(k)})} + \frac {\delta_ {k} ^ {2}}{\rho_ {s + 1} ^ {2} (\mathbf {X} ^ {(k)})}} \leq \min \left\{\frac {\sigma_ {r + 1} ^ {2} (\mathbf {X} ^ {(k)})}{\varepsilon_ {k} ^ {2}}, \frac {\rho_ {s + 1} ^ {2} (\mathbf {X} ^ {(k)})}{\delta_ {k} ^ {2}} \right\},
$$
$$
\begin{array}{l} \left\| \mathbb {P} _ {T _ {k} ^ {\perp}} (\boldsymbol {\Xi}) \right\| _ {F} ^ {2} \leq 2 \left(\min  \left\{\frac {\sigma_ {r + 1} ^ {2} \left(\mathbf {X} ^ {(k)}\right)}{\varepsilon_ {k} ^ {2}}, \frac {\rho_ {s + 1} ^ {2} \left(\mathbf {X} ^ {(k)}\right)}{\delta_ {k} ^ {2}} \right\} \left\langle \boldsymbol {\Xi}, W _ {\mathbf {X} ^ {(k)}, \varepsilon_ {k}, \delta_ {k}} \boldsymbol {\Xi} \right\rangle\right) \tag {51} \\ + 2 \left\| \left(\mathbb {P} _ {\widetilde {\mathbf {U}}, \widetilde {\mathbf {V}}} ^ {\perp} - \mathbb {P} _ {\mathbf {U}, \mathbf {V}} ^ {\perp}\right) \boldsymbol {\Xi} \right\| _ {F} ^ {2} + 2 \left\| \mathbb {P} _ {S ^ {c}} \boldsymbol {\Xi} \left(\widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} - \mathbf {V V} ^ {*}\right) \right\| _ {F} ^ {2}. \\ \end{array}
$$
Next, we control the last two summands in (51) using matrix perturbation results. Recall that  $\mathbf{U}_{\star} \in \mathbb{R}^{n_1 \times r}$  and  $\mathbf{V}_{\star} \in \mathbb{R}^{n_2 \times r}$  are the singular vector matrices of the reduced singular value decomposition of  $\mathbf{X}_{\star}$ . First observe that
$$
\begin{array}{l} \left\| \left(\mathbb {P} _ {\tilde {\mathbf {U}}, \tilde {\mathbf {V}}} - \mathbb {P} _ {\mathbf {U} _ {\star}, \mathbf {V} _ {\star}}\right) \boldsymbol {\Xi} \right\| _ {F} \\ = \left\| \left(\widetilde {\mathbf {U}} \widetilde {\mathbf {U}} ^ {*} - \mathbf {U} _ {\star} \mathbf {U} _ {\star} ^ {*}\right) \Xi (\mathbf {I d} - \mathbf {V} _ {\star} \mathbf {V} _ {\star} ^ {*}) + \left(\mathbf {I d} - \widetilde {\mathbf {U}} \widetilde {\mathbf {U}} ^ {*}\right) \Xi \left(\widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} - \mathbf {V} _ {\star} \mathbf {V} _ {\star} ^ {*}\right) \right\| _ {F} \\ \leq \left\| \widetilde {\mathbf {U}} \widetilde {\mathbf {U}} ^ {*} - \mathbf {U} _ {\star} \mathbf {U} _ {\star} ^ {*} \right\| \| \boldsymbol {\Xi} \| _ {F} \| \mathbf {I d} - \mathbf {V} _ {\star} \mathbf {V} _ {\star} ^ {*} \| + \left\| \mathbf {I d} - \widetilde {\mathbf {U}} \widetilde {\mathbf {U}} ^ {*} \right\| \| \boldsymbol {\Xi} \| _ {F} \| \widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} - \mathbf {V} _ {\star} \mathbf {V} _ {\star} ^ {*} \| \\ \leq \left(\left\| \widetilde {\mathbf {U}} \widetilde {\mathbf {U}} ^ {*} - \mathbf {U} _ {\star} \mathbf {U} _ {\star} ^ {*} \right\| + \left\| \widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} - \mathbf {V} _ {\star} \mathbf {V} _ {\star} ^ {*} \right\|\right) \| \boldsymbol {\Xi} \| _ {F}. \\ \end{array}
$$
Now note that, by [12, Lemma 1] and [65, Theorem 3.5], we obtain
$$
\begin{array}{l} \left\| \widetilde {\mathbf {U}} \widetilde {\mathbf {U}} ^ {*} - \mathbf {U} _ {\star} \mathbf {U} _ {\star} ^ {*} \right\| + \left\| \widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {*} - \mathbf {V} _ {\star} \mathbf {V} _ {\star} ^ {*} \right\| \leq 2 \left(\| \mathbf {U} _ {\star} \widetilde {\mathbf {U}} _ {\perp} ^ {*} \| + \| \mathbf {V} _ {\star} \widetilde {\mathbf {V}} _ {\perp} ^ {*} \|\right) \leq 4 \frac {\| \mathsf {H} _ {s} (\mathbf {X} ^ {(k)}) - \mathbf {X} _ {\star} \|}{\sigma_ {r} (\mathbf {X} _ {\star}) - \sigma_ {r + 1} (\mathsf {H} _ {s} (\mathbf {X} ^ {(k)}))}. \\ \leq 4 \frac {\left\| \mathbf {X} ^ {(k)} - \mathbf {X} _ {\star} \right\|}{\sigma_ {r} \left(\mathbf {X} _ {\star}\right) - \sigma_ {r + 1} \left(\mathbf {X} ^ {(k)}\right)} \leq 4 \frac {\left\| \mathbf {X} ^ {(k)} - \mathbf {X} _ {\star} \right\|}{(1 - 1 / 4 8) \sigma_ {r} \left(\mathbf {X} _ {\star}\right)}, \tag {52} \\ \end{array}
$$
where we used some small observations in the last two inequalities: First,  $\sigma_{r + 1}(\mathsf{H}_s(\mathbf{X}^{(k)}))\leq \sigma_{r + 1}(\mathbf{X}^{(k)})$  which follows from the rectangular Cauchy interlacing theorem [27, Theorem 23]. Second, according to Lemma B.2 and (25), the row-support  $S$  of  $\mathsf{H}_s(\mathbf{X}^{(k)})$  coincides with the row-support  $S_{\star} = \{i\in [n_1]:$ $\| (\mathbf{X}_{\star})_{i,:}\| _2\neq 0\}$  of  $\mathbf{X}_{\star}$  and hence  $\mathsf{H}_s(\mathbf{X}^{(k)}) - \mathbf{X}_{\star}$  is a submatrix of  $\mathbf{X}^{(k)} - \mathbf{X}_{\star}$ . Finally,  $\sigma_{r + 1}(\mathbf{X}^{(k)}) =$ $\| \mathsf{T}_r(\mathbf{X}^{(k)}) - \mathbf{X}^{(k)}\| \leq \| \mathbf{X}_{\star} - \mathbf{X}^{(k)}\| \leq \frac{1}{48}\sigma_r(\mathbf{X}_{\star})$  due to (25). Consequently,
$$
\left\| \left(\mathbb {P} _ {\tilde {\mathbf {U}}, \tilde {\mathbf {V}}} - \mathbb {P} _ {\mathbf {U} _ {\star}, \mathbf {V} _ {\star}}\right) \boldsymbol {\Xi} \right\| _ {F} \leq 4 \frac {\| \mathbf {X} ^ {(k)} - \mathbf {X} _ {\star} \|}{(1 - 1 / 4 8) \sigma_ {r} (\mathbf {X} _ {\star})} \| \boldsymbol {\Xi} \| _ {F}
$$
and, by a similar argument,
$$
\| (\mathbb {P} _ {\mathbf {U} _ {\star}, \mathbf {V} _ {\star}} - \mathbb {P} _ {\mathbf {U}, \mathbf {V}}) \boldsymbol {\Xi} \| _ {F} \leq 4 \frac {\| \mathbf {X} ^ {(k)} - \mathbf {X} _ {\star} \|}{(1 - 1 / 4 8) \sigma_ {r} (\mathbf {X} _ {\star})} \| \boldsymbol {\Xi} \| _ {F},
$$
such that it follows from  $\mathbb{P}_{\mathbf{U},\mathbf{V}}^{\perp} = \mathbf{Id} - \mathbb{P}_{\mathbf{U},\mathbf{V}}$  and  $\mathbb{P}_{\mathbf{U},\tilde{\mathbf{V}}}^{\perp} = \mathbf{Id} - \mathbb{P}_{\tilde{\mathbf{U}},\tilde{\mathbf{V}}}$  that