SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \left\| \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {\top} \widehat {\beta} - \left(\mathbf {Z} _ {f} ^ {\prime}\right) ^ {\top} \beta^ {*} \right\| _ {2} ^ {2} \\ = \| (\widehat {\mathbf {Z}} _ {f} ^ {\prime}) ^ {\top} \widehat {\beta} - [ \mathbf {Z} _ {y} ] _ {L \cdot} \| _ {2} ^ {2} - \| [ \mathbf {Z} _ {x} ] _ {L \cdot} \| _ {2} ^ {2} + 2 \left((\widehat {\mathbf {Z}} _ {f} ^ {\prime}) ^ {\top} \widehat {\beta} - (\mathbf {Z} _ {f} ^ {\prime}) ^ {\top} \beta^ {*}\right) ^ {\top} [ \mathbf {Z} _ {x} ] _ {L}. \\ \leq \| (\widehat {\mathbf {Z}} _ {f} ^ {\prime}) ^ {\top} \beta^ {*} - [ \mathbf {Z} _ {y} ] _ {L \cdot} \| _ {2} ^ {2} - \| [ \mathbf {Z} _ {x} ] _ {L \cdot} \| _ {2} ^ {2} + 2 \left((\widehat {\mathbf {Z}} _ {f} ^ {\prime}) ^ {\top} \widehat {\beta} - (\mathbf {Z} _ {f} ^ {\prime}) ^ {\top} \beta^ {*}\right) ^ {\top} [ \mathbf {Z} _ {x} ] _ {L}. \\ = \| (\widehat {\mathbf {Z}} _ {f} ^ {\prime}) ^ {\top} \beta^ {*} - [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} - [ \mathbf {Z} _ {f} ] _ {L ^ {\cdot}} \| _ {2} ^ {2} - \| [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} \| _ {2} ^ {2} + 2 \left((\widehat {\mathbf {Z}} _ {f} ^ {\prime}) ^ {\top} \widehat {\beta} - (\mathbf {Z} _ {f} ^ {\prime}) ^ {\top} \beta^ {*}\right) ^ {\top} [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} \\ = \left\| \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime} - \mathbf {Z} _ {f} ^ {\prime}\right) ^ {\top} \beta^ {*} - [ \mathbf {Z} _ {x} ] _ {L}. \right\| _ {2} ^ {2} - \| [ \mathbf {Z} _ {x} ] _ {L}. \| _ {2} ^ {2} + 2 \left(\left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {\top} \widehat {\beta} - \left(\mathbf {Z} _ {f} ^ {\prime}\right) ^ {\top} \beta^ {*}\right) ^ {\top} [ \mathbf {Z} _ {x} ] _ {L}. \\ = \left\| \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime} - \mathbf {Z} _ {f} ^ {\prime}\right) ^ {\top} \beta^ {*} \right\| _ {2} ^ {2} + 2 \left(\left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {\top} \widehat {\beta} - \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {\top} \beta^ {*}\right) ^ {\top} [ \mathbf {Z} _ {x} ] _ {L}. \tag {83} \\ \end{array}
$$
Note that  $(\widehat{\mathbf{Z}}_f')^\top \widehat{\beta} = \widetilde{\mathbf{V}} \widetilde{\Sigma} \widetilde{\mathbf{U}}^\top \widehat{\beta} = \widetilde{\mathbf{V}} \widetilde{\Sigma} \widetilde{\mathbf{U}}^\top \widetilde{\mathbf{U}} \widetilde{\Sigma}^\dagger \widetilde{\mathbf{V}}^\top [\mathbf{Z}_y]_{L'} = \widetilde{\mathbf{V}} \widetilde{\mathbf{V}}^\top [\mathbf{Z}_y]_{L'}$ , and  $(\widehat{\mathbf{Z}}_f')^\top \beta^* = \widetilde{\mathbf{V}} \widetilde{\Sigma} \widetilde{\mathbf{U}}^\top \beta^*$ , thus,
$$
\begin{array}{l} \left(\left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {\top} \widehat {\beta} - \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {\top} \beta^ {*}\right) ^ {\top} [ \mathbf {Z} _ {x} ] _ {L.} = [ \mathbf {Z} _ {y} ] _ {L.} ^ {\top} \widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {\top} [ \mathbf {Z} _ {x} ] _ {L.} - \beta^ {* \top} \widetilde {\mathbf {U}} \widetilde {\boldsymbol {\Sigma}} \widetilde {\mathbf {V}} ^ {\top} [ \mathbf {Z} _ {x} ] _ {L}. \\ = \left(\left[ \mathbf {Z} _ {y} \right] _ {L} ^ {\top} \widetilde {\mathbf {V}} - \beta^ {* \top} \widetilde {\mathbf {U}} \widetilde {\boldsymbol {\Sigma}}\right) \widetilde {\mathbf {V}} ^ {\top} \left[ \mathbf {Z} _ {x} \right] _ {L}. \\ = \left(\left[ \mathbf {Z} _ {f} \right] _ {L ^ {\cdot}} ^ {\top} \widetilde {\mathbf {V}} + \left[ \mathbf {Z} _ {x} \right] _ {L ^ {\cdot}} ^ {\top} \widetilde {\mathbf {V}} - \beta^ {* ^ {\top}} \widetilde {\mathbf {U}} \widetilde {\boldsymbol {\Sigma}}\right) \widetilde {\mathbf {V}} ^ {\top} \left[ \mathbf {Z} _ {x} \right] _ {L ^ {\cdot}} \\ = \left(\beta^ {* ^ {\top}} \mathbf {Z} _ {f} ^ {\prime} \widetilde {\mathbf {V}} - \beta^ {* ^ {\top}} \widehat {\mathbf {Z}} _ {f} ^ {\prime} \widetilde {\mathbf {V}}\right) \widetilde {\mathbf {V}} ^ {\top} [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} + \left\| [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} ^ {\top} \widetilde {\mathbf {V}} \right\| _ {2} ^ {2} \\ \leq \left\| \beta^ {* ^ {\top}} \mathbf {Z} _ {f} ^ {\prime} - \beta^ {* ^ {\top}} \widehat {\mathbf {Z}} _ {f} ^ {\prime} \right\| _ {2} \left\| \widetilde {\mathbf {V}} ^ {\top} [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} \right\| _ {2} + \left\| [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} ^ {\top} \widetilde {\mathbf {V}} \right\| _ {2} ^ {2} \\ \leq \left\| \mathbf {Z} _ {f} ^ {\prime} - \widehat {\mathbf {Z}} _ {f} ^ {\prime} \right\| _ {2, \infty} \| \beta^ {*} \| _ {1} \left\| [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} ^ {\top} \widetilde {\mathbf {V}} \right\| _ {2} + \left\| [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} ^ {\top} \widetilde {\mathbf {V}} \right\| _ {2} ^ {2} \tag {84} \\ \end{array}
$$
Finally, from (82) and (83) we get
$$
\begin{array}{l} \| \widetilde {\mathbf {U}} ^ {\top} (\widehat {\beta} - \beta^ {*}) \| _ {2} ^ {2} \leq \frac {2}{\sigma_ {k} (\widehat {\mathbf {Z}} _ {f} ^ {\prime}) ^ {2}} \left(2 \| \mathbf {Z} _ {f} ^ {\prime} - \widehat {\mathbf {Z}} _ {f} ^ {\prime} \| _ {2, \infty} ^ {2} \| \beta^ {*} \| _ {1} ^ {2} \right. \\ \left. + \left\| \mathbf {Z} _ {f} ^ {\prime} - \widehat {\mathbf {Z}} _ {f} ^ {\prime} \right\| _ {2, \infty} \| \beta^ {*} \| _ {1} \left\| [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} ^ {\top} \widetilde {\mathbf {V}} \right\| _ {2} + \left\| [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} ^ {\top} \widetilde {\mathbf {V}} \right\| _ {2} ^ {2}\right) \\ \leq \frac {4}{\sigma_ {k} \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {2}} \left(2 \| \mathbf {Z} _ {f} ^ {\prime} - \widehat {\mathbf {Z}} _ {f} ^ {\prime} \| _ {2, \infty} ^ {2} \| \beta^ {*} \| _ {1} ^ {2} + \left\| \left[ \mathbf {Z} _ {x} \right] _ {L ^ {.}} ^ {\top} \widetilde {\mathbf {V}} \right\| _ {2} ^ {2}\right). \\ \end{array}
$$
The term  $\left\| [\mathbf{Z}_x]_{L^{\cdot}}^{\top}\widetilde{\mathbf{V}}\right\| _2^2$  can be further decomposed as
$$
\left\| \left[ \mathbf {Z} _ {x} \right] _ {L ^ {\cdot}} ^ {\top} \widetilde {\mathbf {V}} \right\| _ {2} ^ {2} \leq 2 \left\| \left[ \mathbf {Z} _ {x} \right] _ {L ^ {\cdot}} ^ {\top} \left(\widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {\top} - \mathbf {V V} ^ {\top}\right) \right\| _ {2} ^ {2} + 2 \left\| \left[ \mathbf {Z} _ {x} \right] _ {L ^ {\cdot}} ^ {\top} \mathbf {V V} ^ {\top} \right\| _ {2} ^ {2}
$$
Using Wedin sin  $\Theta$  Theorem,
$$
\left\| \widetilde {\mathbf {V}} \widetilde {\mathbf {V}} ^ {\top} - \mathbf {V V} ^ {\top} \right\| _ {2} \leq \frac {\left\| \mathbf {Z} _ {x} ^ {\prime} \right\| _ {2}}{\sigma_ {k} \left(\mathbf {Z} _ {f} ^ {\prime}\right)}
$$
$$
\left. \right.\left\| \widetilde {\mathbf {U}} ^ {\top} \left(\widehat {\beta} - \beta^ {*}\right)\right\| _ {2} ^ {2} \leq \frac {8}{\sigma_ {k} \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {2}} \left(\left\| \mathbf {Z} _ {f} ^ {\prime} - \widehat {\mathbf {Z}} _ {f} ^ {\prime} \right\| _ {2, \infty} ^ {2} \left\| \beta^ {*} \right\| _ {1} ^ {2} + \frac {\left\| \mathbf {Z} _ {x} ^ {\prime} \right\| _ {2} ^ {2}}{\sigma_ {k} \left(\mathbf {Z} _ {f} ^ {\prime}\right) ^ {2}} \left\| [ \mathbf {Z} _ {x} ] _ {L \cdot} \right\| _ {2} ^ {2} + \left\| [ \mathbf {Z} _ {x} ] _ {L \cdot} ^ {\top} \mathbf {V} \right\| _ {2} ^ {2}\right). \tag {85}
$$
From (76), (79), and (85) we have
$$
\begin{array}{l} \| \widehat {\beta} - \beta^ {*} \| _ {2} ^ {2} \leq \frac {\| \mathbf {Z} _ {x} ^ {\prime} \| _ {2} ^ {2}}{\sigma_ {k} \left(\mathbf {Z} _ {f} ^ {\prime}\right) ^ {2}} \left(\| \beta^ {*} \| _ {2} ^ {2} + 8 \frac {\| [ \mathbf {Z} _ {x} ] _ {L \cdot} \| _ {2} ^ {2}}{\sigma_ {k} \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {2}}\right) \\ + \frac {8}{\sigma_ {k} \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {2}} \left(2 \| \mathbf {Z} _ {f} ^ {\prime} - \widehat {\mathbf {Z}} _ {f} ^ {\prime} \| _ {2, \infty} ^ {2} \| \beta^ {*} \| _ {1} ^ {2} + \left\| [ \mathbf {Z} _ {x} ] _ {L} ^ {\top} \mathbf {V} \right\| _ {2} ^ {2}\right). \tag {86} \\ \end{array}
$$
F.3 High probability bound
Let  $q \coloneqq NT / L$  be the number of columns in the stacked page matrix  $\widehat{\mathbf{Z}}_f'$ , and let  $C > 0$  be some positive absolute constant. Define
$$
E _ {1} ^ {\prime} := \left\{\| \mathbf {Z} _ {x} ^ {\prime} \| _ {2} \leq C \sigma_ {x} \sqrt {q} \right\},
$$
$$
E _ {2} ^ {\prime} := \left\{\| [ \mathbf {Z} _ {x} ] _ {L ^ {\cdot}} \| _ {2} \leq C \sigma_ {x} \sqrt {q} \right\},
$$
$$
E _ {3} ^ {\prime} := \left\{\| \mathbf {Z} _ {f} ^ {\prime} - \widehat {\mathbf {Z}} _ {f} ^ {\prime} \| _ {2, \infty} ^ {2} \leq \frac {C \sigma_ {x} ^ {2} q ^ {2}}{\sigma_ {k} (\mathbf {Z} _ {f}) ^ {2}} \left(\sigma_ {x} ^ {2} + f _ {\max } ^ {2}\right) + C \sigma_ {x} ^ {2} k \log (q) \right\}
$$
$$
E _ {4} ^ {\prime} := \left\{\left\| \left[ \mathbf {Z} _ {x} \right] _ {L ^ {\cdot}} ^ {\top} \mathbf {V} \right\| _ {2} \leq C \sigma_ {x} \sqrt {k \log (q)} \right\}.
$$
Lemma F.1. For some positive constant  $c_1 > 0$  and  $C > 0$  large enough in definitions of  $E_1', E_2', E_3'$  and  $E_4'$ ,
$$
\mathbb {P} \left(E _ {1} ^ {\prime}\right) \geq 1 - 2 e ^ {- c _ {1} q},
$$
$$
\mathbb {P} \left(E _ {2} ^ {\prime}\right) \geq 1 - 2 e ^ {- c _ {1} q},
$$
$$
\mathbb {P} \left(E _ {3} ^ {\prime}\right) \geq 1 - \frac {c _ {1}}{(N T) ^ {1 1}},
$$
$$
\mathbb {P} \left(E _ {4} ^ {\prime}\right) \geq 1 - \frac {c _ {1}}{\left(N T\right) ^ {1 1}}.
$$
Bounding  $E_1'$  and  $E_2'$ . Note that  $\mathbf{Z}_x'$  is a sub-matrix of  $\mathbf{Z}_x$ , and its operator norm is bounded by that of  $\mathbf{Z}_x$ . Hence, as the probability is an immediate consequence of Lemma A.2. The second event probability follows from the fact that  $\|[\mathbf{Z}_x]_{L'}\|_2 \leq \| \mathbf{Z}_x\|_2$ .
Bounding  $E_3'$ . This is immediate from Corollary C.1 and its consequence in (27).
Bounding  $E_4'$ . First, recall that  $[\mathbf{Z}_x]_L$  is a  $q$ -dimensional sub-gaussian vector with variance proxy  $\sigma_x^2$ . That is, for any unit vector  $\mathbf{v} \in S^{q-1}$ ,  $[\mathbf{Z}_x]_L \cdot \mathbf{v} \sim \mathrm{subG}(\sigma_x^2)$ . Now, recall that  $\mathbf{V} \in \mathbb{R}^{q \times k}$  is a set of  $k$  orthonormal vectors  $\mathbf{V}_i$ ,  $i \in [k]$ . Let  $Z_i = [\mathbf{Z}_x]_L \cdot \mathbf{V}_i$ . Clearly,  $Z_i \sim \mathrm{subG}(\sigma_x^2)$ . Therefore,  $[\mathbf{Z}_x]_L \cdot \mathbf{V}$  is a  $k$ -dimensional vector with dependent sub-gaussian entries with variance proxy  $\sigma_x^2$ . Using Lemma H.4, we have
$$
\mathbb {P} \left(\| [ \mathbf {Z} _ {x} ] _ {L ^ {.}} \mathbf {V} \| _ {2} > t\right) \leq c \exp \left(- \frac {t ^ {2}}{1 6 k \sigma_ {x} ^ {2}}\right).
$$
Therefore, for choice of  $t = C\sigma_x\sqrt{k\log(q)}$  with large enough constant  $C \geq 19$ , we have,
$$
\mathbb {P} \Big (\| [ \mathbf {Z} _ {x} ] _ {L ^ {.}} \mathbf {V} \| _ {2} > C \sigma_ {x} \sqrt {k \log (q)} \Big) \leq \frac {c}{q ^ {2 2}}.
$$
Recalling that  $q > L$ , and  $q = NT / L$ , concludes the proof.
Let  $E' \coloneqq E_1' \cap E_2' \cap E_3' \cap E_4'$ . Then, under event  $E$ , and using (86), we have, with probability  $1 - \frac{c}{(NT)^{11}}$ ,
$$
\begin{array}{l} \left\| \widehat {\beta} - \beta^ {*} \right\| _ {2} ^ {2} \leq \frac {C \sigma_ {x} ^ {2}}{\sigma_ {k} \left(\mathbf {Z} _ {f} ^ {\prime}\right) ^ {2}} \left(q \left\| \beta^ {*} \right\| _ {2} ^ {2} + \frac {q}{\sigma_ {k} \left(\widehat {\mathbf {Z}} _ {f} ^ {\prime}\right) ^ {2}}\right) \\ + \frac {C \sigma_ {x} ^ {2}}{\sigma_ {k} (\widehat {\mathbf {Z}} _ {f} ^ {\prime}) ^ {2}} \left(\left(\frac {q ^ {2}}{\sigma_ {k} (\mathbf {Z} _ {f}) ^ {2}} \left(\sigma_ {x} ^ {2} + f _ {\mathrm {m a x}} ^ {2}\right) + k \log (q)\right) \max \{\| \beta^ {*} \| _ {1} ^ {2}, 1 \}\right). \\ \end{array}
$$
Choosing  $L = \sqrt{NT}$ , and  $k = RG$ , recalling Assumption 4.1, and using Weyl's lemma (H.3) to bound  $|\sigma_k(\widehat{\mathbf{Z}}_f')^2 - \sigma_k(\mathbf{Z}_f')^2|$  we get,
$$
\begin{array}{l} \| \widehat {\beta} - \beta^ {*} \| _ {2} ^ {2} \leq \frac {C \gamma^ {2} \sigma_ {x} ^ {2} R G}{N T} \bigg (\sqrt {N T} \| \beta^ {*} \| _ {2} ^ {2} + R G \log (N T) \\ \left. + \left(\gamma^ {2} R G \left(\sigma_ {x} ^ {2} + f _ {\max } ^ {2}\right) + R G \log (N T)\right) \| \beta^ {*} \| _ {1} ^ {2}\right) \\ \leq \frac {C \gamma^ {4} f _ {\operatorname* {m a x}} ^ {2} \sigma_ {x} ^ {4} R ^ {2} G ^ {2} \log (N T)}{\sqrt {N T}} \max  \{\| \beta^ {*} \| _ {1} ^ {2}, 1 \}, \\ \end{array}
$$
which concludes the proof of Theorem 4.3.
G Proof of Theorem 4.4
In this section, we prove an upper bound on the forecasting error,
$$
\mathsf {F o r E r r} (N, T) = \frac {1}{N T} \sum_ {n = 1} ^ {N} \sum_ {t = T + 1} ^ {2 T} \left(\widehat {y} _ {n} (t) - \bar {y} _ {n} (t)\right) ^ {2},
$$
where  $\bar{y}_n(t) = \mathbb{E}[y_n(t)|y_n(1),\ldots ,y_n(t - 1)] = f_n(t) + \bar{x}_n(t)$ , where  $\bar{x}_n(t) \coloneqq \mathbb{E}[x_n(t)|x_n(1),\ldots ,x_n(t - 1)]$ . First, recall that for  $t > T$
$$
\widehat {y} _ {n} (t) = \widehat {f} _ {n} (t) + \widehat {x} _ {n} (t) = \widehat {\beta} ^ {\top} Y _ {n} (t - 1) + \widehat {\alpha} _ {n} ^ {\top} \widetilde {X} _ {n} (t - 1),
$$
where  $Y_{n}(t - 1) = [y_{n}(t - (L - 1)),\ldots ,y_{n}(t - 1)]$ , and  $\widetilde{X}_n(t - 1) = [\widetilde{x}_n(t - p),\dots ,\widetilde{x}_n(t - 1)]$ . Then, we have,
$$
\frac {1}{N T} \sum_ {n = 1} ^ {N} \sum_ {t = T + 1} ^ {2 T} (\widehat {y} _ {n} (t) - y _ {n} (t)) ^ {2} \leq \frac {2}{N T} \sum_ {n = 1} ^ {N} \sum_ {t = T + 1} ^ {2 T} (\widehat {f} _ {n} (t) - f _ {n} (t)) ^ {2} + (\widehat {x} _ {n} (t) - \bar {x} _ {n} (t)) ^ {2}.
$$
Now, we will bound each error term separately.
G.1 Bounding Forecasting Error of  $f$
To bound the forecasting error of  $f$ , we use the following Lemma.
Lemma G.1. Let the conditions of Theorem 4.1 hold. Then, with probability  $1 - \frac{c}{(NT)^{11}}$
$$
\frac {1}{N T} \sum_ {n = 1} ^ {N} \sum_ {t = T + 1} ^ {2 T} \left(\widehat {f _ {n}} (t) - f _ {n} (t)\right) ^ {2} \leq C (f _ {\max }, \gamma) R ^ {3} G ^ {3} \sigma_ {x} ^ {6} \left(\frac {\max \{\| \beta^ {*} \| _ {1} ^ {2} , 1 \} \log (N T)}{\sqrt {N T}}\right),
$$
where  $c$  is an absolute constant, and  $C(f_{\max}, \gamma)$  denotes a constant that depends only (polynomially) on model parameters  $f_{\max}, \gamma$ .
Proof. The proof of this lemma is similar to that in [3], but we adapt it for our different settings. First, note that, where we assume that  $NT / L$  is an integer,
$$
\frac {1}{N T} \sum_ {n = 1} ^ {N} \sum_ {t = T + 1} ^ {2 T} \left(\widehat {f} _ {n} (t) - f _ {n} (t)\right) ^ {2} = \frac {1}{L} \sum_ {\ell = 1} ^ {L} \frac {L}{N T} \sum_ {n = 1} ^ {N} \sum_ {m = 0} ^ {T / L - 1} \left(\widehat {f} _ {n} (T + m L + \ell) - f _ {n} (T + m L + \ell)\right) ^ {2}.
$$
In what follows, we provide an upper bound for the inner sum  $\frac{L}{NT}\sum_{n = 1}^{N}\sum_{m = 0}^{T / L - 1}\left(\widehat{f}_n(T + mL + \ell) - f_n(T + mL + \ell)\right)^2$  for all  $\ell \in [L]$ . Next, we show, without loss of generality, the case for  $\ell = 1$ . To establish this bound, we first define (and recall) the following notations,
- Recall that  $\mathbf{Z}_y = [\mathbf{Z}(y_1, L, T) \quad \mathbf{Z}(y_2, L, T) \quad \ldots \quad \mathbf{Z}(y_n, L, T)]$  is the stacked Page matrix of  $y_1(t), \ldots, y_n(t)$  for  $t \in [T]$ .
- Similarly, let  $\mathbf{Z}_{y,out}$  denote the stacked Page matrix of the out-of-sample observations  $y_{1}(t),\ldots ,y_{n}(t)$  for  $t\in \{T - L + 2,\dots ,2T - L + 1\}$ , i.e.,  $\mathbf{Z}_{y,out}\in \mathbb{R}^{L\times NT / L}$ .
- Let  $\mathbf{Z}_x$  be the stacked Page matrix of  $x_1(t),\ldots ,x_n(t)$  for  $t\in [T]$ , and  $\mathbf{Z}_{x,out}$  be the stacked Page matrix of  $x_{1}(t),\dots,x_{n}(t)$  for  $t\in \{T - L + 2,\dots,2T - L + 1\}$ .
- Similarly, let  $\mathbf{Z}_f$  and  $\mathbf{Z}_{f,out}$  be defined similarly but for  $f_{1}(t),\ldots ,f_{n}(t)$ .
- Let  $\mathbf{Z}_y' \in \mathbb{R}^{(L - 1) \times (NT / L)}$ , be the a sub-matrix obtained by dropping the  $L$ -th row from the stacked Page matrix  $\mathbf{Z}_y$ . Define  $\mathbf{Z}_{y,out}', \mathbf{Z}_f', \mathbf{Z}_{f,out}', \mathbf{Z}_x'$ , and  $\mathbf{Z}_{x,out}'$  analogously.
- Let  $\mathbf{U} \boldsymbol{\Sigma} \mathbf{V}^{\top}$  and  $\mathbf{U}_{out} \boldsymbol{\Sigma}_{out} \mathbf{V}_{out}^{\top}$  denote the SVD of  $\mathbf{Z}_f'$  and  $\mathbf{Z}_{f,out}'$  respectively.
- let  $\mathbf{V}^{\perp}$  and  $\mathbf{U}^{\perp}$  be matrices of orthonormal basis vectors that span the null space of  $\mathbf{Z}_f'$  and  $\mathbf{Z}_f'^\top$ , respectively. Define  $\mathbf{V}_{out}^{\perp}$  and  $\mathbf{U}_{out}^{\perp}$  similarly for the matrix  $\mathbf{Z}_{f,out}^{\prime}$ .
- Let  $\widetilde{\mathbf{U}}\widetilde{\boldsymbol{\Sigma}}\widetilde{\mathbf{V}}^{\top}$  denote the top  $k$  singular components of the SVD of  $\mathbf{Z}_y^\prime$ , while  $\widetilde{\mathbf{U}}^{\perp}\widetilde{\boldsymbol{\Sigma}}^{\perp}(\widetilde{\mathbf{V}}^{\perp})^{\top}$  denote the remaining  $L - k - 1$  components such that  $\mathbf{Z}_y^\prime = \widetilde{\mathbf{U}}\widetilde{\boldsymbol{\Sigma}}\widetilde{\mathbf{V}}^{\top} + \widetilde{\mathbf{U}}^{\perp}\widetilde{\boldsymbol{\Sigma}}^{\perp}(\widetilde{\mathbf{V}}^{\perp})^{\top}$ .
- Similarly, Let  $\widetilde{\mathbf{U}}_{out}\widetilde{\boldsymbol{\Sigma}}_{out}\widetilde{\mathbf{V}}_{out}^{\top}$  denote the top  $k$  singular components of the SVD of  $\mathbf{Z}_{y,out}^{\prime}$ , while  $\widetilde{\mathbf{U}}_{out}^{\perp}\widetilde{\boldsymbol{\Sigma}}_{out}^{\perp}(\widetilde{\mathbf{V}}_{out}^{\perp})^{\top}$  denote the remaining  $L - k - 1$  components such that  $\mathbf{Z}_{y,out}^{\prime} = \widetilde{\mathbf{U}}_{out}\widetilde{\boldsymbol{\Sigma}}_{out}\widetilde{\mathbf{V}}_{out}^{\top} + \widetilde{\mathbf{U}}_{out}^{\perp}\widetilde{\boldsymbol{\Sigma}}_{out}^{\perp}(\widetilde{\mathbf{V}}_{out}^{\perp})^{\top}$ .
Let  $\widehat{\mathbf{Z}}_f = \widetilde{\mathbf{U}}\widetilde{\boldsymbol{\Sigma}}\widetilde{\mathbf{V}}^\top$  and  $\widehat{\mathbf{Z}}_{f,out} = \widetilde{\mathbf{U}}_{out}\widetilde{\boldsymbol{\Sigma}}_{out}\widetilde{\mathbf{V}}_{out}^\top$
Recall that we are interested in a high-probability bound for the following out-of-sample prediction error:
$$
\frac {L}{N T} \sum_ {n = 1} ^ {N} \sum_ {m = 0} ^ {T / L - 1} \left(\widehat {f} _ {n} (T + m L + 1) - f _ {n} (T + m L + 1)\right) ^ {2}. \tag {87}
$$
Recall that  $\widehat{f}_n(t) = \widehat{\beta}^\top Y_n(t - 1)$  and hence we can rewrite (87) as
$$
\begin{array}{l} \sum_ {n = 1} ^ {N} \sum_ {m = 0} ^ {T / L} \left(\widehat {f} _ {n} (T + m L + 1) - f _ {n} (T + m L + 1)\right) ^ {2} = \left\| \mathbf {Z} _ {y, o u t} ^ {\prime} ^ {\top} \widehat {\beta} - \left[ \mathbf {Z} _ {f, o u t} \right] _ {L \cdot} ^ {\top} \right\| _ {2} ^ {2} (88) \\ = \left\| \mathbf {Z} _ {y, o u t} ^ {\prime} ^ {\top} \widehat {\beta} - \mathbf {Z} _ {f, o u t} ^ {\prime} ^ {\top} \beta^ {*} \right\| _ {2} ^ {2} (89) \\ \end{array}
$$
Next, we derive a deterministic upper bound for the expression above.
Deterministic Bound. Through adding and subtracting  $\widehat{\mathbf{Z}}_{f,out}^{\top}\widehat{\beta}$  and triangle inequality, we have
$$
\left\| \left(\mathbf {Z} _ {y, o u t} ^ {\prime}\right) ^ {\top} \widehat {\beta} - \left(\mathbf {Z} _ {f, o u t} ^ {\prime}\right) ^ {\top} \beta^ {*} \right\| _ {2} \leq \left\| \left(\mathbf {Z} _ {y, o u t} ^ {\prime} - \widehat {\mathbf {Z}} _ {f, o u t}\right) ^ {\top} \widehat {\beta} \right\| _ {2} + \left\| \widehat {\mathbf {Z}} _ {f, o u t} ^ {\top} \widehat {\beta} - \left(\mathbf {Z} _ {f, o u t} ^ {\prime}\right) ^ {\top} \beta^ {*} \right\| _ {2} \tag {90}
$$
Next, we proceed to bound each of the two terms on the right hand side.
First term:  $\left\| (\mathbf{Z}_{y,out}^{\prime} - \widehat{\mathbf{Z}}_{f,out})^{\top}\widehat{\boldsymbol{\beta}}\right\| _2$
$$
\begin{array}{l} \left\| \left(\mathbf {Z} _ {y, o u t} ^ {\prime} - \widehat {\mathbf {Z}} _ {f, o u t}\right) ^ {\top} \widehat {\beta} \right\| _ {2} ^ {2} = \| \widetilde {\mathbf {V}} _ {o u t} ^ {\perp} \widetilde {\boldsymbol {\Sigma}} _ {o u t} ^ {\perp} \left(\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}\right) ^ {\top} \widehat {\beta} \| _ {2} ^ {2} \\ \leq \left\| \tilde {\boldsymbol {\Sigma}} _ {o u t} ^ {\perp} \right\| _ {2} ^ {2} \left\| \left(\tilde {\mathbf {U}} _ {o u t} ^ {\perp}\right) ^ {\top} \hat {\beta} \right\| _ {2} ^ {2}. \tag {91} \\ \end{array}
$$
Note that  $\| \widetilde{\Sigma}_{out}^{\perp}\|_{2}$  equals the  $(k + 1)$ -th singular value of  $\mathbf{Z}_{y,out}^{\prime}$ . Using Weyl's inequality (see Lemma H.3), and recalling that  $\mathbf{Z}_{y,out}^{\prime} = \mathbf{Z}_{f,out}^{\prime} + \mathbf{Z}_{x,out}^{\prime}$ , and that the rank of  $\mathbf{Z}_{f,out}^{\prime}$  is  $k$ , we can bound the  $(k + 1)$ -th singular value of  $\mathbf{Z}_{y,out}^{\prime}$  by the operator norm of  $\mathbf{Z}_{x,out}^{\prime}$ . That is,
$$
\left\| \widetilde {\boldsymbol {\Sigma}} _ {o u t} ^ {\perp} \right\| _ {2} ^ {2} \leq \left\| \mathbf {Z} _ {x, o u t} ^ {\prime} \right\| _ {2} ^ {2}. \tag {92}
$$
Next, we bound the term  $\| (\widetilde{\mathbf{U}}_{out}^{\perp})^{\top}\widehat{\beta}\| _2^2$
$$
\begin{array}{l} \| (\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}) ^ {\top} \widehat {\beta} \| _ {2} ^ {2} = \| \widetilde {\mathbf {U}} _ {o u t} ^ {\perp} (\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}) ^ {\top} \widehat {\beta} \| _ {2} ^ {2} \\ = \| \widetilde {\mathbf {U}} _ {o u t} ^ {\perp} (\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}) ^ {\top} \beta^ {*} + \widetilde {\mathbf {U}} _ {o u t} ^ {\perp} (\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}) ^ {\top} (\widehat {\beta} - \beta^ {*}) \| _ {2} ^ {2} \\ \leq 2 \| \widetilde {\mathbf {U}} _ {o u t} ^ {\perp} \left(\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}\right) ^ {\top} \beta^ {*} \| _ {2} ^ {2} + 2 \| \widehat {\beta} - \beta^ {*} \| _ {2} ^ {2}. \tag {93} \\ \end{array}
$$
Further expanding the first term,
$$
\begin{array}{l} \left\| \widetilde {\mathbf {U}} _ {o u t} ^ {\perp} \left(\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}\right) ^ {\top} \beta^ {*} \right\| _ {2} = \left\| \widetilde {\mathbf {U}} _ {o u t} ^ {\perp} \left(\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}\right) ^ {\top} \mathbf {U} _ {o u t} \left(\mathbf {U} _ {o u t}\right) ^ {\top} \beta^ {*} \right\| _ {2} \\ \leq \left\| \mathbf {U} _ {o u t} ^ {\perp} \left(\mathbf {U} _ {o u t} ^ {\perp}\right) ^ {\top} \mathbf {U} _ {o u t} \left(\mathbf {U} _ {o u t}\right) ^ {\top} \beta^ {*} \right\| _ {2} \\ + \left\| \left(\widetilde {\mathbf {U}} _ {o u t} ^ {\perp} \left(\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}\right) ^ {\top} - \mathbf {U} _ {o u t} ^ {\perp} \left(\mathbf {U} _ {o u t} ^ {\perp}\right) ^ {\top}\right) \mathbf {U} _ {o u t} \left(\mathbf {U} _ {o u t}\right) ^ {\top} \beta^ {*} \right\| _ {2} \\ \leq \left\| \left(\widetilde {\mathbf {U}} _ {o u t} ^ {\perp} \left(\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}\right) ^ {\top} - \mathbf {U} _ {o u t} ^ {\perp} \left(\mathbf {U} _ {o u t} ^ {\perp}\right) ^ {\top}\right) \beta^ {*} \right\| _ {2} \\ \leq \left\| \widetilde {\mathbf {U}} _ {o u t} ^ {\perp} \left(\widetilde {\mathbf {U}} _ {o u t} ^ {\perp}\right) ^ {\top} - \mathbf {U} _ {o u t} ^ {\perp} \left(\mathbf {U} _ {o u t} ^ {\perp}\right) ^ {\top} \right\| _ {2} \| \beta^ {*} \| _ {2} \\ = \left\| \widetilde {\mathbf {U}} _ {o u t} \widetilde {\mathbf {U}} _ {o u t} ^ {\top} - \mathbf {U} _ {o u t} \mathbf {U} _ {o u t} ^ {\top} \right\| _ {2} \| \beta^ {*} \| _ {2}. \\ \end{array}
$$
Where in the first equality we use the fact that  $\beta^{*} = \mathbf{U}_{out}(\mathbf{U}_{out})^{\top}\beta^{*}$ , i.e.,  $\beta^{*}$  lives in the column space of  $\mathbf{Z}_{f,out}^{\prime}$  (Assumption 4.2).
Next, by Wedin sin  $\Theta$  Theorem (see [10, 32]) we bound  $\left\| \widetilde{\mathbf{U}}_{out} \widetilde{\mathbf{U}}_{out}^{\top} - \mathbf{U}_{out} \mathbf{U}_{out}^{\top} \right\|_2$  as follows: