SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} h _ {t + 1} ^ {j} (\boldsymbol {\alpha}) - h _ {t + 1} ^ {\mathrm {r f}, j} (\boldsymbol {\alpha}) \\ = \phi_ {t + 1} ^ {j} ^ {\top} \boldsymbol {\theta} _ {t + 1} ^ {j} - \phi_ {0} ^ {j} ^ {\top} \bar {\boldsymbol {\theta}} _ {t + 1} ^ {j} \\ = \phi_ {t} ^ {j} ^ {\top} \boldsymbol {\theta} _ {t + 1} ^ {j} + \left(\phi_ {t + 1} ^ {j} ^ {\top} - \phi_ {t} ^ {j} ^ {\top}\right) \boldsymbol {\theta} _ {t + 1} ^ {j} - \phi_ {0} ^ {j} ^ {\top} \bar {\boldsymbol {\theta}} _ {t + 1} ^ {j} \\ = \phi_ {t} ^ {j} ^ {\top} \left(\boldsymbol {\theta} _ {t} ^ {j} - \eta \mathbf {g} _ {t} ^ {j}\right) + \left(\phi_ {t + 1} ^ {j} ^ {\top} - \phi_ {t} ^ {j} ^ {\top}\right) \boldsymbol {\theta} _ {t + 1} ^ {j} - \phi_ {0} ^ {j} ^ {\top} \bar {\boldsymbol {\theta}} _ {t + 1} ^ {j} \\ = \phi_ {t} ^ {j} ^ {\top} \left(\boldsymbol {\theta} _ {t} ^ {j} - \eta \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {j} \left(\boldsymbol {\theta} _ {t} ^ {j}\right) - \eta \left(\mathbf {g} _ {t} ^ {j} - \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {j} \left(\boldsymbol {\theta} _ {t} ^ {j}\right)\right)\right) + \left(\phi_ {t + 1} ^ {j} ^ {\top} - \phi_ {t} ^ {j} ^ {\top}\right) \boldsymbol {\theta} _ {t + 1} ^ {j} - \phi_ {0} ^ {j} ^ {\top} \left(\bar {\boldsymbol {\theta}} _ {t} ^ {j} - \eta \nabla_ {\bar {\boldsymbol {\theta}} _ {t} ^ {j}} \mathcal {R} ^ {\mathrm {r f}, j} \left(\bar {\boldsymbol {\theta}} _ {t} ^ {j}\right)\right) \\ = h _ {t} ^ {j} (\boldsymbol {\alpha}) - h _ {t} ^ {\mathrm {r f}, j} (\boldsymbol {\alpha}) + \left(\phi_ {t + 1} ^ {j} ^ {\top} - \phi_ {t} ^ {j} ^ {\top}\right) \boldsymbol {\theta} _ {t + 1} ^ {j} + \eta \phi_ {t} ^ {j} ^ {\top} \left(\nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} \left(\boldsymbol {\theta} _ {t} ^ {j}\right) - \mathbf {g} _ {t} ^ {j}\right) \\ + \eta \left(\phi_ {0} ^ {j} ^ {\top} \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {\mathrm {r f}, j} \left(\boldsymbol {\theta} _ {t} ^ {j}\right) - \phi_ {t} ^ {j} ^ {\top} \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {j} \left(\boldsymbol {\theta} _ {t} ^ {j}\right)\right). \\ \end{array}
$$
Define  $r_t(\pmb{\alpha}) = h_t^j(\pmb{\alpha}) - w_*(\pmb{\alpha})(j)$ , and  $r_t^{\mathrm{rf}}(\pmb{\alpha}) = h_t^{\mathrm{rf},j}(\pmb{\alpha}) - w_*(\pmb{\alpha})(j)$ . Notice that
$$
\begin{array}{l} \phi_ {0} ^ {j} ^ {\top} \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {\mathrm {r f}, j} (\boldsymbol {\theta} _ {t} ^ {j}) - \phi_ {t} ^ {j} ^ {\top} \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {j} (\boldsymbol {\theta} _ {t} ^ {j}) = \phi_ {0} ^ {j} ^ {\top} (\boldsymbol {\alpha}) \frac {2}{n} \sum_ {i = 1} ^ {n} \phi_ {0} ^ {j} (\boldsymbol {\alpha} _ {i}) r _ {t} ^ {\mathrm {r f}} (\boldsymbol {\alpha} _ {i}) - \phi_ {t} ^ {j} ^ {\top} (\boldsymbol {\alpha}) \frac {2}{n} \sum_ {i = 1} ^ {n} \phi_ {t} (\boldsymbol {\alpha} _ {i}) r _ {t} (\boldsymbol {\alpha} _ {i}) \\ = - \phi_ {0} ^ {j} ^ {\top} (\boldsymbol {\alpha}) \frac {2}{n} \sum_ {i = 1} ^ {n} \phi_ {0} (\boldsymbol {\alpha} _ {i}) \left(h _ {t} (\boldsymbol {\alpha} _ {i}) - h _ {t} ^ {\mathrm {r f}} (\boldsymbol {\alpha} _ {i})\right) \\ + \left(\phi_ {0} ^ {j} ^ {\top} (\boldsymbol {\alpha}) \frac {2}{n} \sum_ {i = 1} ^ {n} \phi_ {0} ^ {j} (\boldsymbol {\alpha} _ {i}) - \phi_ {t} ^ {j} ^ {\top} (\boldsymbol {\alpha}) \frac {2}{n} \sum_ {i = 1} ^ {n} \phi_ {t} ^ {j} (\boldsymbol {\alpha} _ {i})\right) r _ {t} (\boldsymbol {\alpha} _ {i}) \\ \end{array}
$$
We define the following stacked vector representation :
$$
\mathbf {h} _ {t + 1} ^ {j} := \left[ h _ {t + 1} ^ {j} \right. ^ {\top} (\boldsymbol {\alpha} _ {1}), \dots , h _ {t + 1} ^ {j} \left. \right. ^ {\top} (\boldsymbol {\alpha} _ {n}) ] ^ {\top} \in \mathbb {R} ^ {n},
$$
$$
\mathbf {r} _ {t} := \left[ r _ {t} ^ {\top} (\boldsymbol {\alpha} _ {1}), \dots , r _ {t} ^ {\top} (\boldsymbol {\alpha} _ {n}) \right] ^ {\top} \in \mathbb {R} ^ {n}.
$$
Hence we can write the pointwise difference into the following compact form:
$$
\begin{array}{l} \mathbf {h} _ {t + 1} ^ {j} - \mathbf {h} _ {t + 1} ^ {\mathrm {r f}, j} = \left(\mathbf {I} - \frac {2 \eta}{n} \boldsymbol {\Phi} _ {0} ^ {j} ^ {\top} \boldsymbol {\Phi} _ {0} ^ {j}\right) \left(\mathbf {h} _ {t} ^ {j} - \mathbf {h} _ {t} ^ {\mathrm {r f}, j}\right) \\ + \underbrace {(\boldsymbol {\Phi} _ {t + 1} ^ {j} ^ {\top} - \boldsymbol {\Phi} _ {t} ^ {j} ^ {\top}) \boldsymbol {\theta} _ {t + 1} ^ {j}} _ {T _ {1}} \\ + \eta \underbrace {\Phi_ {t} ^ {j}} _ {T _ {2}} ^ {\top} (\nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {j} (\boldsymbol {\theta} _ {t} ^ {j}) - \mathbf {g} _ {t} ^ {j}) \\ + \eta \underbrace {\frac {2}{n} \left(\Phi_ {0} ^ {j} {} ^ {\top} \Phi_ {0} ^ {j} - \Phi_ {t} ^ {j} {} ^ {\top} \Phi_ {t} ^ {j}\right) \mathbf {r} _ {t}} _ {T _ {3}}. \\ \end{array}
$$
Unrolling the recursion to  $t = 0$  yields:
$$
\begin{array}{l} \mathbf {h} _ {t + 1} ^ {j} - \mathbf {h} _ {t + 1} ^ {\mathrm {r f}, j} = \sum_ {s = 0} ^ {t} (\mathbf {I} - \frac {2 \eta}{n} \hat {\mathbf {K}} _ {0} ^ {j}) ^ {t - s} (\boldsymbol {\Phi} _ {s + 1} ^ {j} ^ {\top} - \boldsymbol {\Phi} _ {s} ^ {j} ^ {\top}) \boldsymbol {\theta} _ {t + 1} ^ {j} + \eta \sum_ {s = 0} ^ {t} (\mathbf {I} - \frac {2 \eta}{n} \hat {\mathbf {K}} _ {0} ^ {j}) ^ {t - s} \boldsymbol {\Phi} _ {s} ^ {j} ^ {\top} (\nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} (\boldsymbol {\theta} _ {s} ^ {j}) - \mathbf {g} _ {s} ^ {j}) \\ + \eta \sum_ {s = 0} ^ {t} (\mathbf {I} - \frac {2 \eta}{n} \hat {\mathbf {K}} _ {0} ^ {j}) ^ {t - s} \frac {2}{n} \left(\hat {\mathbf {K}} _ {0} ^ {j} - \hat {\mathbf {K}} _ {s} ^ {j}\right) \mathbf {r} _ {s}. \\ \end{array}
$$
Taking spectral norm on both sides yields:
$$
\begin{array}{l} \left\| \mathbf {h} _ {t + 1} ^ {j} - \mathbf {h} _ {t + 1} ^ {\mathrm {r f}, j} \right\| = \underbrace {\left\| \sum_ {s = 0} ^ {t} (\mathbf {I} - \frac {2 \eta}{n} \hat {\mathbf {K}} _ {0} ^ {j}) ^ {t - s} \left(\boldsymbol {\Phi} _ {s + 1} ^ {j} \right. ^ {\top} - \boldsymbol {\Phi} _ {s} ^ {j} ^ {\top}\right) \boldsymbol {\theta} _ {t + 1} ^ {j}} _ {T _ {1}} \\ + \underbrace {\eta \left\| \sum_ {s = 0} ^ {t} (\mathbf {I} - \frac {2 \eta}{n} \hat {\mathbf {K}} _ {0} ^ {j}) ^ {t - s} \boldsymbol {\Phi} _ {s} ^ {j} ^ {\top} \left(\nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {j} \left(\boldsymbol {\theta} _ {s} ^ {j}\right) - \mathbf {g} _ {s} ^ {j}\right) \right\|} _ {T _ {2}} \\ + \underbrace {\eta \left\| \sum_ {s = 0} ^ {t} (\mathbf {I} - \frac {2 \eta}{n} \hat {\mathbf {K}} _ {0} ^ {j}) ^ {t - s} \frac {2}{n} \left(\hat {\mathbf {K}} _ {0} ^ {j} - \hat {\mathbf {K}} _ {s} ^ {j}\right) \mathbf {r} _ {s} \right\|}. \\ \end{array}
$$
We first bound  $T_{1}$  as follows:
$$
\begin{array}{l} \left\| \sum_ {s = 0} ^ {t} \left(\mathbf {I} - \frac {2 \eta}{n} \hat {\mathbf {K}} _ {0} ^ {j}\right) ^ {t - s} \left(\Phi_ {s + 1} ^ {j} ^ {\top} - \Phi_ {s} ^ {j} ^ {\top}\right) \boldsymbol {\theta} _ {t + 1} ^ {j} \right\| \leq \sum_ {s = 0} ^ {t} \left\| \left(\mathbf {I} - \frac {2 \eta}{n} \hat {\mathbf {K}} _ {0} ^ {j}\right) ^ {t - s} \right\| \left\| \left(\Phi_ {s + 1} ^ {j} ^ {\top} - \Phi_ {s} ^ {j} ^ {\top}\right) \boldsymbol {\theta} _ {t + 1} ^ {j} \right\| \\ \leq \sum_ {s = 0} ^ {t} \left(1 - \frac {2 \eta}{n} \lambda_ {\min } \left(\hat {\mathbf {K}} _ {0} ^ {j}\right)\right) ^ {t - s} \left\| \left(\boldsymbol {\Phi} _ {s + 1} ^ {j} ^ {\top} - \boldsymbol {\Phi} _ {s} ^ {j} ^ {\top}\right) \boldsymbol {\theta} _ {t + 1} ^ {j} \right\| \\ \leq \sum_ {s = 0} ^ {t} \left(1 - \frac {\lambda_ {0} \eta}{n}\right) ^ {t - s} \left\| \left(\boldsymbol {\Phi} _ {s + 1} ^ {j} \mathbin {\text {T}} - \boldsymbol {\Phi} _ {s} ^ {j} \mathbin {\text {T}}\right) \boldsymbol {\theta} _ {t + 1} ^ {j} \right\| \\ \end{array}
$$
where we apply triangle inequality and Cauchy-Schwartz inequality. To bound  $\left\| (\Phi_{s + 1}^{\top} - \Phi_s^{\top})\pmb{\theta}_{t + 1}\right\|$ , we evoke Eq.(28)):
$$
\left\| (\boldsymbol {\Phi} _ {s + 1} ^ {j} ^ {\top} - \boldsymbol {\Phi} _ {s} ^ {j \top}) \boldsymbol {\theta} _ {t + 1} ^ {j} \right\| \leq 2 \sqrt {n} \left(\frac {4 8 n B _ {y}}{\lambda_ {0}} + \sqrt {\nu}\right) \left(\frac {2 \eta \sqrt {\hat {\mathcal {R}} ^ {j} (\boldsymbol {\theta} _ {s} ^ {j})}}{\sqrt {m}} + \frac {2 \eta}{\sqrt {m}} \sqrt {(1 - \mu \gamma) ^ {K s} D}\right)
$$
Now we plug in the convergence rate from Theorem 5, and obtain
$$
\begin{array}{l} \left\| \left(\boldsymbol {\Phi} _ {s + 1} ^ {j} \mathbin {\text {T}} - \boldsymbol {\Phi} _ {s} ^ {j} \mathbin {\text {T}}\right) \boldsymbol {\theta} _ {t + 1} ^ {j} \right\| \\ \leq 2 \sqrt {n} \left(\frac {4 8 n B _ {y}}{\lambda_ {0}} + \sqrt {\nu}\right) \left(\frac {2 \eta \sqrt {\left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {s} \widehat {\mathcal {R}} ^ {j} \left(\boldsymbol {\theta} _ {0} ^ {j}\right) + \frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}}}{\sqrt {m}} + \frac {2 \eta}{\sqrt {m}} \sqrt {(1 - \mu \gamma) ^ {K s} D}\right) \\ \end{array}
$$
$$
\begin{array}{l} T _ {1} \leq 2 \sqrt {n} \left(\frac {4 8 n B _ {y}}{\lambda_ {0}} + \sqrt {\nu}\right) \sum_ {s = 0} ^ {t} \left(1 - \frac {\lambda_ {0} \eta}{n}\right) ^ {t - s} \\ \times \left(\frac {2 \eta \sqrt {\left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {s} \widehat {\mathcal {R}} (\boldsymbol {\theta} _ {0}) + \frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}}}{\sqrt {m}} + \frac {2 \eta}{\sqrt {m}} \sqrt {(1 - \mu \gamma) ^ {K s} D}\right) \\ \leq \frac {4 \eta \sqrt {n}}{\sqrt {m}} \left(\frac {4 8 n B _ {y}}{\lambda_ {0}} + \sqrt {\nu}\right) \sum_ {s = 0} ^ {t} (1 - \frac {\lambda_ {0} \eta}{n}) ^ {t - s} \\ \times \left(\sqrt {\left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {s} \widehat {\mathcal {R}} (\boldsymbol {\theta} _ {0}) + \frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}} + \sqrt {(1 - \mu \gamma) ^ {K s} D}\right) \\ \leq \frac {4 \eta \sqrt {n}}{\sqrt {m}} \left(\frac {4 8 n B _ {y}}{\lambda_ {0}} + \sqrt {\nu}\right) \left(1 - \frac {\lambda_ {0} \eta}{n}\right) ^ {\frac {t}{2}} \\ \times \sum_ {s = 0} ^ {t} (1 - \frac {\lambda_ {0} \eta}{n}) ^ {\frac {t}{2} - s} \left(\left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {s}{2}} \sqrt {\widehat {\mathcal {R}} (\pmb {\theta} _ {0})} + \sqrt {\frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}} + (1 - \mu \gamma) ^ {\frac {s}{2}} \sqrt {(1 - \mu \gamma) ^ {K} D}\right). \\ \end{array}
$$
Using the fact that  $1 - \frac{\lambda_0\eta}{n} \leq 1 - \frac{\lambda_0\eta}{4n}$  and our choice of learning rate such that  $1 - \mu \gamma \leq 1 - \frac{\lambda_0\eta}{4n}$ , we have:
$$
\begin{array}{l} T _ {1} \leq \frac {\sqrt {n} 4 \eta}{\sqrt {m}} \left(\frac {4 8 n B _ {y}}{\lambda_ {0}} + \sqrt {\nu}\right) \\ \times \left((1 - \frac {\lambda_ {0} \eta}{n}) ^ {\frac {t}{2}} \frac {8 n}{\lambda_ {0} \eta} \sqrt {\widehat {\mathcal {R}} ^ {j} (\pmb {\theta} _ {0} ^ {j})} + \frac {4 n}{\lambda_ {0} \eta} \sqrt {\frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}} + (1 - \frac {\lambda_ {0} \eta}{n}) ^ {\frac {t}{2}} \frac {8 n}{\lambda_ {0} \eta} \sqrt {(1 - \mu \gamma) ^ {K} D}\right). \\ \end{array}
$$
For  $T_{2}$ , similarly, we apply triangle and Cauchy-Schwartz inequality:
$$
T _ {2} \leq \eta \sum_ {s = 0} ^ {t} \left(1 - \frac {\eta \lambda_ {0}}{n}\right) ^ {t - s} \left\| \Phi_ {s} ^ {j} ^ {\top} (\nabla_ {\pmb {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {j} (\pmb {\theta} _ {s} ^ {j}) - \mathbf {g} _ {s} ^ {j}) \right\|
$$
To bound  $\left\| \Phi_s^j^\top (\nabla_{\pmb {\theta}_t^j}\widehat{\mathcal{R}}^j (\pmb {\theta}_s^j) - \mathbf{g}_s)\right\|$ , we evoke Lemma 11:
$$
\left\| \boldsymbol {\Phi} _ {s} ^ {j} ^ {\top} (\nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \widehat {\mathcal {R}} ^ {j} (\boldsymbol {\theta} _ {s} ^ {j}) - \mathbf {g} _ {s} ^ {j}) \right\| \leq \frac {2}{n} \left(\left\| \hat {\mathbf {K}} _ {0} ^ {j} \right\| + 2 n ^ {2} (\rho + \sqrt {\frac {\nu}{m}})\right) \sqrt {(1 - \mu \gamma) ^ {K s} D}.
$$
Putting the above inequality back to  $T_{2}$  yields:
$$
\begin{array}{l} T _ {2} \leq \eta \sum_ {s = 0} ^ {t} \left(1 - \frac {\eta \lambda_ {0}}{n}\right) ^ {t - s} \frac {2}{n} \left(\frac {3}{2} \| \mathbf {K} \| + 2 n ^ {2} (\rho + \sqrt {\frac {\nu}{m}})\right) \sqrt {(1 - \mu \gamma) ^ {K s} D} \\ \leq \eta \left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2}} \sum_ {s = 0} ^ {t} \left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2} - \frac {s}{2}} \frac {2}{n} \left(\frac {3}{2} \| \mathbf {K} \| + 2 n ^ {2} \left(\frac {4 8 n B _ {y}}{\lambda_ {0} \sqrt {m}} + \sqrt {\frac {\nu}{m}}\right)\right) \sqrt {(1 - \mu \gamma) ^ {K} D} \\ \leq \left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2}} \frac {1 6}{\lambda_ {0}} \left(\frac {3}{2} \| \mathbf {K} \| + 2 n ^ {2} \left(\frac {4 8 n B _ {y}}{\lambda_ {0} \sqrt {m}} + \sqrt {\frac {\nu}{m}}\right)\right) \sqrt {(1 - \mu \gamma) ^ {K} D} \\ \end{array}
$$
For  $T_{3}$ , we also apply Cauchy-Schwartz and get:
$$
\begin{array}{l} T _ {3} \leq \eta \sum_ {s = 0} ^ {t} \left(1 - \frac {\eta \lambda_ {0}}{n}\right) ^ {t - s} \left\| \frac {2}{n} \left(\hat {\mathbf {K}} _ {0} ^ {j} - \hat {\mathbf {K}} _ {s} ^ {j}\right) \mathbf {r} _ {s} \right\| \\ \leq \eta \frac {2}{n} \sum_ {s = 0} ^ {t} (1 - \frac {\eta \lambda_ {0}}{n}) ^ {t - s} \left\| \left(\hat {\mathbf {K}} _ {0} ^ {j} - \hat {\mathbf {K}} _ {s} ^ {j}\right) \right\| \| \mathbf {r} _ {s} \| \\ \leq \eta \frac {2}{n} \sum_ {s = 0} ^ {t} (1 - \frac {\eta \lambda_ {0}}{n}) ^ {t - s} 2 n ^ {2} (\rho + \sqrt {\frac {\nu}{m}}) \sqrt {n} \sqrt {\hat {\mathcal {R}} ^ {j} \left(\boldsymbol {\theta} _ {t} ^ {j}\right)} \\ \end{array}
$$
where at last step we plug in empirical gram matrix bound from Lemma 10. Now it suffices to plug in bound for  $\widehat{\mathcal{R}} (\pmb {\theta}_t)$
$$
\begin{array}{l} T _ {3} \leq \eta \frac {2}{n} \sum_ {s = 0} ^ {t} (1 - \frac {\eta \lambda_ {0}}{n}) ^ {t - s} 2 n ^ {2} (\rho + \sqrt {\frac {\nu}{m}}) \sqrt {n} \sqrt {\left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {s} \widehat {\mathcal {R}} ^ {j} (\pmb {\theta} _ {0} ^ {j}) + \frac {4 n (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}} \\ \leq \eta \frac {2}{n} 2 n ^ {2} (\rho + \sqrt {\frac {\nu}{m}}) \sqrt {n} \left((1 - \frac {\eta \lambda_ {0}}{n}) ^ {\frac {t}{2}} \sum_ {s = 0} ^ {t} (1 - \frac {\eta \lambda_ {0}}{n}) ^ {\frac {t}{2} - \frac {s}{2}} \sqrt {\widehat {\mathcal {R}} ^ {j} (\boldsymbol {\theta} _ {0} ^ {j})} + \frac {8 n}{\eta \lambda_ {0}} \sqrt {\frac {4 n (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}}\right) \\ \leq 4 \eta \sqrt {n} n \left(\frac {4 8 n B _ {y}}{\lambda_ {0} \sqrt {m}} + \sqrt {\frac {\nu}{m}}\right) \left((1 - \frac {\eta \lambda_ {0}}{4 n}) ^ {\frac {t}{2}} \frac {8 n}{\eta \lambda_ {0}} \sqrt {\widehat {\mathcal {R}} ^ {j} (\boldsymbol {\theta} _ {0} ^ {j})} + \frac {8 n}{\eta \lambda_ {0}} \sqrt {\frac {4 n (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}}\right). \\ \end{array}
$$
$$
\begin{array}{l} \left\| \mathbf {h} _ {t + 1} ^ {j} - \mathbf {h} _ {t + 1} ^ {\mathrm {r f}, j} \right\| \leq \frac {\sqrt {n} 4 \eta}{\sqrt {m}} \left(\frac {4 8 n B _ {y}}{\lambda_ {0}} + \sqrt {\nu}\right) \\ \times \left((1 - \frac {\lambda_ {0} \eta}{4 n}) ^ {\frac {t}{2}} \frac {8 n}{\lambda_ {0} \eta} \sqrt {\hat {\mathcal {R}} ^ {j} (\boldsymbol {\theta} _ {0} ^ {j})} + \frac {4 n}{\lambda_ {0} \eta} \sqrt {\frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}} + (1 - \frac {\lambda_ {0} \eta}{4 n}) ^ {\frac {t}{2}} \frac {8 n}{\lambda_ {0} \eta} \sqrt {(1 - \mu \gamma) ^ {K} D}\right) \\ + \left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2}} \frac {1 6}{\lambda_ {0}} \left(\frac {3}{2} \| \mathbf {K} \| + 2 n ^ {2} \left(\frac {4 8 n B _ {y}}{\lambda_ {0} \sqrt {m}} + \sqrt {\frac {\nu}{m}}\right)\right) \sqrt {(1 - \mu \gamma) ^ {K} D} \\ + 4 \eta \sqrt {n} n \left(\frac {4 8 n B _ {y}}{\lambda_ {0} \sqrt {m}} + \sqrt {\frac {\nu}{m}}\right) \left((1 - \frac {\eta \lambda_ {0}}{4 n}) ^ {\frac {t}{2}} \frac {8 n}{\eta \lambda_ {0}} \sqrt {\widehat {\mathcal {R}} ^ {j} (\boldsymbol {\theta} _ {0} ^ {j})} + \frac {8 n}{\eta \lambda_ {0}} \sqrt {\frac {4 n (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}}\right). \\ \end{array}
$$
$$
\text {L e t} \tilde {n} := \left(\frac {4 8 n B _ {y}}{\lambda_ {0} \sqrt {m}} + \sqrt {\frac {\nu}{m}}\right)
$$
$$
\begin{array}{l} \left\| \mathbf {h} _ {t + 1} ^ {j} - \mathbf {h} _ {t + 1} ^ {\mathrm {r f}, j} \right\| \leq 4 \sqrt {n} \tilde {n} \left(\left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2}} \frac {8 n}{\lambda_ {0}} \sqrt {\hat {\mathcal {R}} ^ {j} (\boldsymbol {\theta} _ {0} ^ {j})} \right. \\ + \frac {4 n}{\lambda_ {0}} \sqrt {\frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}} + \left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2}} \frac {8 n}{\lambda_ {0}} \sqrt {(1 - \mu \gamma) ^ {K} D} \\ + \left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2}} \frac {1 6}{\lambda_ {0}} \left(\frac {3}{2} \| \mathbf {K} \| + 2 n ^ {2} \tilde {n}\right) \sqrt {(1 - \mu \gamma) ^ {K} D} \\ + 4 \sqrt {n} n \tilde {n} \left(\left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2}} \frac {8 n}{\lambda_ {0}} \sqrt {\widehat {\mathcal {R}} ^ {j} \left(\boldsymbol {\theta} _ {0} ^ {j}\right)} + \frac {8 n}{\lambda_ {0}} \sqrt {\frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}}\right) \\ \leq \left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2}} \frac {1 6}{\lambda_ {0}} \left(\left(\frac {3}{2} \| \mathbf {K} \| + 2 n ^ {2} \tilde {n} + 2 \sqrt {n} n \tilde {n}\right) \sqrt {(1 - \mu \gamma) ^ {K} D} + \left(2 \sqrt {n} n ^ {2} \tilde {n} + 2 \sqrt {n} n \tilde {n}\right) \sqrt {\hat {\mathcal {R}} (\boldsymbol {\theta} _ {0})}\right) \\ + \left(2 \sqrt {n} \tilde {n} + 4 \sqrt {n} n \tilde {n}\right) \frac {8 n}{\lambda_ {0}} \sqrt {\frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}}. \\ \end{array}
$$
Lemma 13 (Neural network parameter and RF parameter coupling). For Algorithm 2, the following statement holds for any  $j \in [d]$  and  $t \in [T]$ :
$$
\begin{array}{l} \left\| \boldsymbol {\theta} _ {t} ^ {j} - \bar {\boldsymbol {\theta}} _ {t} ^ {j} \right\| ^ {2} \leq O \left(\frac {n ^ {2}}{\lambda_ {0} ^ {2}} \left(\frac {4 8 n B _ {y}}{\lambda_ {0} \sqrt {m}} + \sqrt {\frac {\nu}{m}}\right) + \frac {n ^ {6}}{\lambda_ {0} ^ {4}} \left(\frac {4 8 n B _ {y}}{\lambda_ {0} \sqrt {m}} + \sqrt {\frac {\nu}{m}}\right) ^ {2}\right) \\ + O \left(\eta^ {3} T ^ {2} \left(\frac {4 8 n B _ {y}}{\lambda_ {0} \sqrt {m}} + \sqrt {\frac {\nu}{m}}\right) ^ {2} \frac {n ^ {3} (1 - \frac {1}{\kappa}) ^ {K} D}{\lambda_ {0} ^ {3}} (\lambda_ {\max} + \lambda_ {0}) ^ {2} + \frac {(1 - \frac {1}{\kappa}) ^ {K} D}{\lambda_ {0} ^ {2}}\right). \\ \end{array}
$$
Proof. According to updating rule of neural network and random feature predictor, we have
$$
\begin{array}{l} \boldsymbol {\theta} _ {t + 1} ^ {j} - \bar {\boldsymbol {\theta}} _ {t + 1} ^ {j} \\ = \boldsymbol {\theta} _ {t} ^ {j} - \bar {\boldsymbol {\theta}} _ {t} ^ {j} - \eta \left(\mathbf {g} _ {t} ^ {j} - \nabla_ {\bar {\boldsymbol {\theta}} _ {t} ^ {j}} \mathcal {R} ^ {\mathrm {r f}, j} \left(\bar {\boldsymbol {\theta}} _ {t} ^ {j}\right)\right) \\ = \boldsymbol {\theta} _ {t} ^ {j} - \bar {\boldsymbol {\theta}} _ {t} ^ {j} - \eta \frac {2}{n} \sum_ {i = 1} ^ {n} (\phi_ {t} ^ {j} (\boldsymbol {\alpha} _ {i}) \mathbf {r} _ {t} ^ {j} (\boldsymbol {\alpha} _ {i}) - \phi_ {0} ^ {j} (\boldsymbol {\alpha} _ {i}) \mathbf {r} _ {t} ^ {\mathrm {r f}, j} (\boldsymbol {\alpha} _ {i})) - \eta (\mathbf {g} _ {t} ^ {j} - \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \mathcal {R} ^ {j} (\bar {\boldsymbol {\theta}} _ {t} ^ {j})) \\ = \boldsymbol {\theta} _ {t} ^ {j} - \bar {\boldsymbol {\theta}} _ {t} ^ {j} - \eta \frac {2}{n} \sum_ {i = 1} ^ {n} \left(\phi_ {t} ^ {j} (\boldsymbol {\alpha} _ {i}) \mathbf {r} _ {t} ^ {j} (\boldsymbol {\alpha} _ {i}) - \phi_ {0} (\boldsymbol {\alpha} _ {i}) \mathbf {r} _ {t} ^ {\mathrm {r f}, j} (\boldsymbol {\alpha} _ {i})\right) - \eta \left(\mathbf {g} _ {t} ^ {j} - \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \mathcal {R} ^ {j} \left(\bar {\boldsymbol {\theta}} _ {t} ^ {j}\right)\right) \\ = \boldsymbol {\theta} _ {t} ^ {j} - \bar {\boldsymbol {\theta}} _ {t} ^ {j} - \eta \frac {2}{n} \sum_ {i = 1} ^ {n} (\phi_ {t} ^ {j} (\boldsymbol {\alpha} _ {i}) - \phi_ {0} ^ {j} (\boldsymbol {\alpha} _ {i})) \mathbf {r} _ {t} (\boldsymbol {\alpha} _ {i}) - \eta \frac {2}{n} \sum_ {i = 1} ^ {n} \phi_ {0} ^ {j} (\boldsymbol {\alpha} _ {i}) (\mathbf {r} _ {t} (\boldsymbol {\alpha} _ {i}) - \mathbf {r} _ {t} ^ {\mathrm {r f}} (\boldsymbol {\alpha} _ {i})) - \eta (\mathbf {g} _ {t} ^ {j} - \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \mathcal {R} (\bar {\boldsymbol {\theta}} _ {t} ^ {j})) \\ = \pmb {\theta} _ {t} ^ {j} - \bar {\pmb {\theta}} _ {t} ^ {j} - \eta \frac {2}{n} (\pmb {\Phi} _ {t} ^ {j} (\pmb {\alpha} _ {i}) - \pmb {\Phi} _ {0} ^ {j} (\pmb {\alpha} _ {i})) \mathbf {r} _ {t} - \eta \frac {2}{n} \pmb {\Phi} _ {0} ^ {j} (\pmb {\alpha} _ {i}) (\mathbf {h} _ {t} ^ {j} - \mathbf {h} _ {t} ^ {\mathrm {r f}, j}) - \eta (\mathbf {g} _ {t} ^ {j} - \nabla_ {\pmb {\theta} _ {t} ^ {j}} \mathcal {R} ^ {j} (\bar {\pmb {\theta}} _ {t} ^ {j})). \\ \end{array}
$$
Now taking norm on both side yields:
$$
\left\| \boldsymbol {\theta} _ {t + 1} ^ {j} - \bar {\boldsymbol {\theta}} _ {t + 1} ^ {j} \right\| = \left\| \boldsymbol {\theta} _ {t} ^ {j} - \bar {\boldsymbol {\theta}} _ {t} ^ {j} \right\| + \eta \frac {2}{n} \underbrace {\left\| (\boldsymbol {\Phi} _ {t} ^ {j} - \boldsymbol {\Phi} _ {0} ^ {j}) \mathbf {r} _ {t} ^ {j} \right\|} _ {T _ {1}} + \eta \frac {2}{n} \underbrace {\left\| \boldsymbol {\Phi} _ {0} ^ {j} (\boldsymbol {\alpha} _ {i}) (\mathbf {h} _ {t} ^ {j} - \mathbf {h} _ {t} ^ {\mathrm {r f} , j}) \right\|} _ {T _ {2}} + \eta \underbrace {\left\| \mathbf {g} _ {t} ^ {j} - \nabla_ {\boldsymbol {\theta} _ {t} ^ {j}} \mathcal {R} (\bar {\boldsymbol {\theta}} _ {t} ^ {j}) \right\|}.
$$
$$
\begin{array}{l} T _ {1} \leq \left\| \left(\Phi_ {t} ^ {j} - \Phi_ {0} ^ {j}\right) \right\| \left\| \mathbf {r} _ {t} ^ {j} \right\| \\ \leq \sqrt {n (\rho + \sqrt {\frac {\nu}{m}})} \sqrt {n} \sqrt {\widehat {\mathcal {R}} ^ {j} (\pmb {\theta} _ {t} ^ {j})} \\ \leq \sqrt {n (\rho + \sqrt {\frac {\nu}{m}})} \sqrt {n} \sqrt {\left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {t} \widehat {\mathcal {R}} ^ {j} \left(\boldsymbol {\theta} _ {0} ^ {j}\right) + \frac {4 n (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}}. \\ \end{array}
$$
where we plug in results from Lemma 10 and Theorem 5.
For  $T_{2}$ :
$$
\begin{array}{l} T _ {2} \leq \left\| \boldsymbol {\Phi} _ {0} ^ {j} \right\| \left\| \mathbf {h} _ {t} ^ {j} - \mathbf {h} _ {t} ^ {\mathrm {r f}, j} \right\| \\ \leq \sqrt {n} \left(1 - \frac {\eta \lambda_ {0}}{4 n}\right) ^ {\frac {t}{2}} \frac {1 6}{\lambda_ {0}} \left(\left(\frac {3}{2} \| \mathbf {K} \| + 2 n ^ {2} \tilde {n} + 2 \sqrt {n} n \tilde {n}\right) \sqrt {(1 - \mu \gamma) ^ {K} D} + \left(2 \sqrt {n} n ^ {2} \tilde {n} + 2 \sqrt {n} n \tilde {n}\right) \sqrt {\hat {\mathcal {R}} (\boldsymbol {\theta} _ {0})}\right) \\ + \sqrt {n} \left(2 \sqrt {n} \tilde {n} + 4 \sqrt {n} n \tilde {n}\right) \frac {8 n}{\lambda_ {0}} \sqrt {\frac {4 n C (1 - \mu \gamma) ^ {K} D}{\eta \lambda_ {0}}}. \\ \end{array}
$$
For  $T_{3}$ , we evoke Lemma 11:
$$
T _ {3} \leq 2 \sqrt {(1 - \mu \gamma) ^ {K t} D} \leq 2 \sqrt {(1 - \frac {n \lambda_ {0}}{4 n}) ^ {t} (1 - \mu \gamma) ^ {K} D}.
$$
Putting pieces together we have: