SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \left\| y _ {0} \left(\boldsymbol {\theta} _ {1}\right) - y _ {0} \left(\boldsymbol {\theta} _ {2}\right) \right\| \\ = \left\| r (s, 1) - r (s, 0) + \sum_ {s ^ {\prime}} P (s ^ {\prime} | s, 1) \max  _ {a} f _ {0} (\boldsymbol {\theta} _ {1}; \boldsymbol {\phi} (s ^ {\prime}, a)) - \sum_ {s ^ {\prime}} P (s ^ {\prime} | s, 0) \max  _ {a} f _ {0} (\boldsymbol {\theta} _ {1}; \boldsymbol {\phi} (s ^ {\prime}, a)) \right. \\ \left. - r (s, 1) - r (s, 0) + \sum_ {s ^ {\prime}} P \left(s ^ {\prime} \mid s, 1\right) \max  _ {a} f _ {0} \left(\boldsymbol {\theta} _ {2}; \boldsymbol {\phi} \left(s ^ {\prime}, a\right)\right) - \sum_ {s ^ {\prime}} P \left(s ^ {\prime} \mid s, 0\right) \max  _ {a} f _ {0} \left(\boldsymbol {\theta} _ {2}; \boldsymbol {\phi} \left(s ^ {\prime}, a\right)\right) \right\rVert \\ = \left\| \sum_ {s ^ {\prime}} P \left(s ^ {\prime} \mid s, 1\right) \max  _ {a} f _ {0} \left(\boldsymbol {\theta} _ {1}; \boldsymbol {\phi} \left(s ^ {\prime}, a\right)\right) - \sum_ {s ^ {\prime}} P \left(s ^ {\prime} \mid s, 1\right) \max  _ {a} f _ {0} \left(\boldsymbol {\theta} _ {2}; \boldsymbol {\phi} \left(s ^ {\prime}, a\right)\right) \right. \\ \left. - \sum_ {s ^ {\prime}} P \left(s ^ {\prime} \mid s, 0\right) \max  _ {a} f _ {0} \left(\boldsymbol {\theta} _ {1}; \boldsymbol {\phi} \left(s ^ {\prime}, a\right)\right) + \sum_ {s ^ {\prime}} P \left(s ^ {\prime} \mid s, 0\right) \max  _ {a} f _ {0} \left(\boldsymbol {\theta} _ {2}; \boldsymbol {\phi} \left(s ^ {\prime}, a\right)\right) \right\rVert \\ \leq \left\| \sum_ {s ^ {\prime}} P \left(s ^ {\prime} \mid s, 1\right) \max  _ {a} f _ {0} \left(\boldsymbol {\theta} _ {1}; \boldsymbol {\phi} \left(s ^ {\prime}, a\right)\right) - \sum_ {s ^ {\prime}} P \left(s ^ {\prime} \mid s, 1\right) \max  _ {a} f _ {0} \left(\boldsymbol {\theta} _ {2}; \boldsymbol {\phi} \left(s ^ {\prime}, a\right)\right) \right\| \\ + \left\| \sum_ {s ^ {\prime}} P (s ^ {\prime} | s, 0) \max  _ {a} f _ {0} (\boldsymbol {\theta} _ {1}; \boldsymbol {\phi} (s ^ {\prime}, a)) + \sum_ {s ^ {\prime}} P (s ^ {\prime} | s, 0) \max  _ {a} f _ {0} (\boldsymbol {\theta} _ {2}; \boldsymbol {\phi} (s ^ {\prime}, a)) \right\| \\ \leq 2 \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|, \\ \end{array}
$$
with the last inequality holds due to (27) and (28).
Proof. 1) We first show that there exists a constant  $\mu_1 > 0$  such that  $\mathbb{E}[\hat{\pmb{\theta}}^{\mathsf{T}}h_0(X,\pmb {\theta},\lambda)]\leq -\mu_1\| \hat{\pmb{\theta}}\|^2$ . According to the definition of  $\pmb{\theta}_0^*$  given in Definition 1,  $\mathbb{E}[h_0(X,\pmb{\theta}_0^*,y_0(\pmb{\theta}_0^*))] = 0$ . Hence, we have
$$
\begin{array}{l} \mathbb {E} \left[ \hat {\boldsymbol {\theta}} ^ {\intercal} \left(h _ {0} (X, \boldsymbol {\theta}, \lambda) - h _ {0} \left(X, \boldsymbol {\theta} _ {0} ^ {*}, y _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right)\right) \right] \right. \\ = \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ h _ {0} (X, \boldsymbol {\theta}, \lambda) - h _ {0} \left(X, \boldsymbol {\theta} _ {0} ^ {*}, y _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right)\right) \right] \\ = \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) \left[ r (s, a) + (1 - a) \lambda - I _ {0} (\boldsymbol {\theta}) + \max  _ {a _ {1}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} \left(s ^ {\prime}, a _ {1}\right)) - f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) \right] \right. \\ \left. - \nabla_ {\boldsymbol {\theta}} f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}; \phi (s, a)\right) \left[ r (s, a) + (1 - a) y _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right) - I _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right) + \max  _ {a _ {2}} f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}; \phi \left(s ^ {\prime}, a _ {2}\right)\right) - f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}; \phi (s, a)\right) \right] \right] \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(b _ {1})} {=} \hat {\boldsymbol {\theta}} ^ {\intercal} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) \left[ (1 - a) \left(\lambda - y _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right)\right) + I \left(\boldsymbol {\theta} _ {0} ^ {*}\right) - I _ {0} (\boldsymbol {\theta}) + f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}; \boldsymbol {\phi} (s, a)\right) - f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) \right. \right. \\ \left. \left. + \max  _ {a _ {1}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s ^ {\prime}, a _ {1})) - \max  _ {a _ {2}} f _ {0} (\boldsymbol {\theta} _ {0} ^ {*}; \boldsymbol {\phi} (s ^ {\prime}, a _ {2})) \right] \right] \\ = \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) \left[ \max  _ {a _ {1}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} \left(s ^ {\prime}, a _ {1}\right)) - \max  _ {a _ {2}} f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}; \boldsymbol {\phi} \left(s ^ {\prime}, a _ {2}\right)\right) \right] \right] \\ \left. - \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) [ I _ {0} (\boldsymbol {\theta}) - I _ {0} (\boldsymbol {\theta} _ {0} ^ {*}) ] \right] - \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) [ f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) - f _ {0} (\boldsymbol {\theta} _ {0} ^ {*}; \boldsymbol {\phi} (s, a)) ] \right] \right] \\ + \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) [ (1 - a) (\lambda - y _ {0} (\boldsymbol {\theta} _ {0} ^ {*}))) ] \right] \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(b _ {2})} {\leq} \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) \max  _ {a ^ {\prime}} \left[ f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s ^ {\prime}, a ^ {\prime})) - f _ {0} (\boldsymbol {\theta} _ {0} ^ {*}; \boldsymbol {\phi} (s ^ {\prime}, a ^ {\prime})) \right] \right] \\ \left. - \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) [ I _ {0} (\boldsymbol {\theta}) - I _ {0} (\boldsymbol {\theta} _ {0} ^ {*}) ] \right] - \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) [ f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) - f _ {0} (\boldsymbol {\theta} _ {0} ^ {*}; \boldsymbol {\phi} (s, a)) ] \right] \right] \\ + \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) [ (1 - a) (\lambda - y _ {0} (\boldsymbol {\theta} _ {0} ^ {*}))) ] \right] \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(b _ {3})} {\leq} \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) \max  _ {a ^ {\prime}} \left[ f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s ^ {\prime}, a ^ {\prime})) - f _ {0} (\boldsymbol {\theta} _ {0} ^ {*}; \boldsymbol {\phi} (s ^ {\prime}, a ^ {\prime})) \right] \right] \\ \left. - \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) [ I _ {0} (\boldsymbol {\theta}) - I _ {0} (\boldsymbol {\theta} _ {0} ^ {*}) ] \right] - \hat {\boldsymbol {\theta}} ^ {\mathsf {T}} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) [ f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) - f _ {0} (\boldsymbol {\theta} _ {0} ^ {*}; \boldsymbol {\phi} (s, a)) ] \right] \right] \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(b _ {4})} {=} \| \hat {\boldsymbol {\theta}} \| ^ {2} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) ^ {\intercal} \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s ^ {\prime}, \tilde {a})) \right] \\ \left. \right. - \| \hat {\boldsymbol {\theta}} \| ^ {2} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) ^ {\top} \left[ \frac {1}{2 S} \sum_ {\tilde {s} \in S} \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (\tilde {s}, 0)) + f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (\tilde {s}, 1)) \right]\right] \\ - \| \hat {\boldsymbol {\theta}} \| ^ {2} \mathbb {E} \left[ \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) ^ {\top} \nabla_ {\boldsymbol {\theta}} f _ {0} (\boldsymbol {\theta}; \boldsymbol {\phi} (s, a)) \right] \\ \end{array}
$$
where  $(b_{1})$  holds since  $\nabla_{\pmb{\theta}}f_0(\pmb {\theta};\pmb {\phi}(s,a)) = \nabla_{\pmb{\theta}}f_0(\pmb{\theta}_0^*;\pmb {\phi}(s,a))$  as in (26),  $(b_{2})$  is due to the fact that  $\max_{a_1}f_0(\pmb {\theta};\pmb {\phi}(s',a_1)) - \max_{a_1}f_0(\pmb{\theta}_0^*;\pmb {\phi}(s',a_2))\leq \max_{a'}\left[f_0(\pmb {\theta};\pmb {\phi}(s',a')) - f_0(\pmb{\theta}_0^*;\pmb {\phi}(s',a'))\right]$  and  $(b_{3})$  holds due to the fact that  $\hat{\pmb{\theta}}^{\mathsf{T}}\mathbb{E}\Big[\nabla_{\pmb{\theta}}f_0(\pmb {\theta};\pmb {\phi}(s,a))[(1 - a)(\lambda -y_0(\pmb{\theta}_0^*))]\Big]\leq 0$  since a larger Whittle index  $\lambda$  will choose the action  $a = 1$ . Notice that the  $\tilde{a}$  in  $(b_{4})$  represents the action  $a^\prime$  which maximizes  $f_{0}(\pmb {\theta};\pmb {\phi}(s^{\prime},a^{\prime})) - f_{0}(\pmb{\theta}_{0}^{*};\pmb {\phi}(s^{\prime},a^{\prime}))$ . Due to the definition of  $\nabla_{\pmb{\theta}}f_0(\pmb {\theta};\pmb {\phi}(s,a))$  in (26), we show that  $\mathbb{E}[\hat{\pmb{\theta}}^{\mathsf{T}}h_0(X,\pmb {\theta},\lambda)]\leq 0$ .
2) Next, we show that there exists a constant  $\mu_2 > 0$  such that  $\mathbb{E}[\hat{\lambda} g_0(X,\pmb {\theta},\lambda)]\leq -\mu_2\| \hat{\lambda}\| ^2$ . According to the definition of  $g_{0}(\pmb {\theta})$ , i.e.,  $g_{0}(\pmb {\theta})\coloneqq f_{0}(\pmb {\theta};\pmb {\phi}(s,1)) - f_{0}(\pmb {\theta};\pmb {\phi}(s,0))$ . Since  $y_{0}(\pmb {\theta})$  is the solution of  $\lambda$  such that  $f_{0}(\pmb {\theta};\pmb {\phi}(s,1)) = f_{0}(\pmb {\theta};\pmb {\phi}(s,0))$ , the signs of  $\hat{\lambda}\coloneqq \lambda -y_0(\theta)$  and  $f_{0}(\pmb {\theta};\pmb {\phi}(s,1)) - f_{0}(\pmb {\theta};\pmb {\phi}(s,0))$  are always opposite. Hence, we have  $\mathbb{E}[\hat{\lambda} g_0(X,\pmb {\theta},\lambda)]\leq 0$ , which completes the proof.
Proof. Under Lemma 1, we have
$$
\left\| h _ {0} (X, \boldsymbol {\theta}, \lambda) - h _ {0} (X, \boldsymbol {\theta} ^ {*}, \lambda^ {*}) \right\| \leq 3 \| \boldsymbol {\theta} - \boldsymbol {\theta} ^ {*} \| + \| \boldsymbol {\lambda} - \boldsymbol {\lambda} ^ {*} \|. \tag {29}
$$
Let  $L = \max (3,\max_{X}h_{0}(X,\pmb{\theta}^{*},\lambda^{*}))$ , then according to (29), we have
$$
\left\| h _ {0} (X, \boldsymbol {\theta}, \lambda) \right\| \leq L \left(\left\| \boldsymbol {\theta} - \boldsymbol {\theta} ^ {*} \right\| + \left\| \boldsymbol {\lambda} - \boldsymbol {\lambda} ^ {*} \right\| + 1\right).
$$
Denote  $h_0^i (X,\pmb {\theta},\lambda)$  as the  $i$ -th element of  $h_0(X,\pmb {\theta},\lambda)$ . Following [15], we can show that  $\pmb {\theta}\in \mathbb{R}^{md}$ ,  $\lambda \in \mathbb{R}^1$ , and  $x\in \mathcal{X}$ ,
$$
\begin{array}{l} \| \mathbb {E} [ h _ {0} (X _ {k}, \boldsymbol {\theta}, \lambda) | X _ {0} = x ] - \mathbb {E} _ {\mu} [ h _ {0} (X, \boldsymbol {\theta}, \lambda) ] \| \\ \leq \sum_ {i = 1} ^ {m d} | \mathbb {E} [ h _ {i} (X _ {k}, \boldsymbol {\theta}, \lambda) | X _ {0} = x ] - \mathbb {E} _ {\mu} [ h _ {0} ^ {i} (X, \boldsymbol {\theta}, \lambda) ] | \\ \leq 2 L (\| \pmb {\theta} - \pmb {\theta} ^ {*} \| + \| \lambda - \lambda^ {*} \| + 1) \sum_ {i = 1} ^ {m d} \left| \mathbb {E} \left[ \frac {h _ {0} ^ {i} (X _ {k} , \pmb {\theta} , \lambda)}{2 L (\| \pmb {\theta} - \pmb {\theta} ^ {*} \| + \| \lambda - \lambda^ {*} \| + 1)} \right| X _ {0} = x \right] \\ - \mathbb {E} _ {\mu} \left[ \frac {h _ {0} ^ {i} (X , \boldsymbol {\theta} , \lambda)}{2 L (\| \boldsymbol {\theta} - \boldsymbol {\theta} ^ {*} \| + \| \lambda - \lambda^ {*} \| + 1)} \right] \\ \leq 2 L \left(\| \boldsymbol {\theta} - \boldsymbol {\theta} ^ {*} \| + \| \lambda - \lambda^ {*} \| + 1\right) m d C \rho^ {k}, \\ \end{array}
$$
where the last inequality holds due to Assumption 1. To guarantee  $2L(\|\pmb{\theta} - \pmb{\theta}^{*}\| + \|\lambda - \lambda^{*}\| + 1) mdC\rho^{k} \leq \delta (\|\pmb{\theta} - \pmb{\theta}^{*}\| + \|\lambda - \lambda^{*}\| + 1)$ , we have
$$
\tau_ {\delta} \leq \frac {\log (1 / \delta) + \log (2 L C m d)}{\log (1 / \rho)},
$$
Proof. Based on the definition of  $M(\pmb{\theta}_k, \lambda_k)$  in (15), we have
$$
\begin{array}{l} M (\boldsymbol {\theta} _ {k}, \lambda_ {k}) := \frac {\eta_ {k}}{\alpha_ {k}} \| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} ^ {*} \| ^ {2} + \| \lambda_ {k} - y (\boldsymbol {\theta} _ {k}) \| ^ {2} \\ = \frac {\eta_ {k}}{\alpha_ {k}} \| \pmb {\theta} _ {k} - \pmb {\theta} _ {0} ^ {*} + \pmb {\theta} _ {0} ^ {*} - \pmb {\theta} ^ {*} \| ^ {2} + \| \lambda_ {k} - y _ {0} (\pmb {\theta} _ {k}) + y _ {0} (\pmb {\theta} _ {k}) - y (\pmb {\theta} _ {k}) \| ^ {2} \\ \leq \frac {2 \eta_ {k}}{\alpha_ {k}} \left(\| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} _ {0} ^ {*} \| ^ {2} + \| \boldsymbol {\theta} _ {0} ^ {*} - \boldsymbol {\theta} ^ {*} \| ^ {2}\right) + 2 \left(\| \lambda_ {k} - y _ {0} (\boldsymbol {\theta} _ {k}) \| ^ {2} + \| y _ {0} (\boldsymbol {\theta} _ {k}) - y (\boldsymbol {\theta} _ {k}) \| ^ {2}\right) \\ = 2 \hat {M} (\boldsymbol {\theta} _ {k}, \lambda_ {k}) + \frac {2 \eta_ {k}}{\alpha_ {k}} \| \boldsymbol {\theta} _ {0} ^ {*} - \boldsymbol {\theta} ^ {*} \| ^ {2} + 2 \| y _ {0} (\boldsymbol {\theta} _ {k}) - y (\boldsymbol {\theta} _ {k}) \| ^ {2} \\ \leq 2 \hat {M} (\boldsymbol {\theta} _ {k}, \lambda_ {k}) + \frac {2 \eta_ {k} c _ {0} ^ {2}}{\alpha_ {k}} \| \operatorname {s p a n} \left(f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right) - f \left(\boldsymbol {\theta} ^ {*}\right)\right) \| ^ {2} + 2 \| y _ {0} (\boldsymbol {\theta} _ {k}) - y (\boldsymbol {\theta} _ {k}) \| ^ {2}, \tag {30} \\ \end{array}
$$
where the first inequality holds based on  $\| \pmb{x} + \pmb{y}\|^2 \leq 2\| \pmb{x}\|^2 + 2\| \pmb{y}\|^2$ , and the second inequality holds based on Assumption 3. Next, we bound  $\| \text{span}(f_0(\pmb{\theta}_0^*) - f(\pmb{\theta}^*))\|$  as follows
$$
\begin{array}{l} \| s p a n \left(f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right) - f \left(\boldsymbol {\theta} ^ {*}\right)\right) \| = \| s p a n \left(f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right) - \Pi_ {\mathcal {F}} f \left(\boldsymbol {\theta} ^ {*}\right) + \Pi_ {\mathcal {F}} f \left(\boldsymbol {\theta} ^ {*}\right) - f \left(\boldsymbol {\theta} ^ {*}\right)\right) \| \\ \leq \left\| s p a n \left(f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right) - \Pi_ {\mathcal {F}} f \left(\boldsymbol {\theta} ^ {*}\right)\right) \right\| + \left\| s p a n \left(\Pi_ {\mathcal {F}} f \left(\boldsymbol {\theta} ^ {*}\right) - f \left(\boldsymbol {\theta} ^ {*}\right)\right) \right\| \\ = \| \operatorname {s p a n} \left(\Pi_ {\mathcal {F}} \mathcal {T} f _ {0} \left(\boldsymbol {\theta} ^ {*}\right) - \Pi_ {\mathcal {F}} \mathcal {T} f \left(\boldsymbol {\theta} ^ {*}\right)\right) \| + \| \operatorname {s p a n} \left(\Pi_ {\mathcal {F}} f \left(\boldsymbol {\theta} ^ {*}\right) - f \left(\boldsymbol {\theta} ^ {*}\right)\right) \| \\ \leq \kappa \| \operatorname {s p a n} \left(f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right) - f \left(\boldsymbol {\theta} ^ {*}\right)\right) \| + \| \operatorname {s p a n} \left(\Pi_ {\mathcal {F}} f \left(\boldsymbol {\theta} ^ {*}\right) - f \left(\boldsymbol {\theta} ^ {*}\right)\right) \|, \tag {31} \\ \end{array}
$$
where the last inequality follows (24). This indicates that
$$
\left\| \operatorname {s p a n} \left(f _ {0} \left(\boldsymbol {\theta} _ {0} ^ {*}\right) - f \left(\boldsymbol {\theta} ^ {*}\right)\right) \right\| ^ {2} \leq \frac {1}{(1 - \kappa) ^ {2}} \left\| \operatorname {s p a n} \left(\Pi_ {\mathcal {F}} f \left(\boldsymbol {\theta} ^ {*}\right) - f \left(\boldsymbol {\theta} ^ {*}\right)\right) \right\| ^ {2}. \tag {32}
$$
We further bound  $\| y_0(\pmb{\theta}_k) - y(\pmb{\theta}_k) \|^2$  as follows
$$
\begin{array}{l} \| y _ {0} (\boldsymbol {\theta} _ {k}) - y (\boldsymbol {\theta} _ {k}) \| ^ {2} = \left\| \sum_ {s ^ {\prime}} p \left(s ^ {\prime} \mid s, 1\right) \max  _ {a _ {1}} f _ {0} \left(\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} \left(s ^ {\prime}, a _ {1}\right)\right) - \sum_ {s ^ {\prime}} p \left(s ^ {\prime} \mid s, 0\right) \max  _ {a _ {2}} f _ {0} \left(\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} \left(s ^ {\prime}, a _ {2}\right)\right) \right. \\ \left. - \sum_ {s ^ {\prime}} p \left(s ^ {\prime} \mid s, 1\right) \max  _ {a _ {3}} f \left(\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} \left(s ^ {\prime}, a _ {3}\right)\right) + \sum_ {s ^ {\prime}} p \left(s ^ {\prime} \mid s, 0\right) \max  _ {a _ {4}} f \left(\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} \left(s ^ {\prime}, a _ {4}\right)\right) \right\rVert^ {2} \\ = \left\| \sum_ {s ^ {\prime}} p \left(s ^ {\prime} \mid s, 1\right) \left(\max  _ {a _ {1}} f _ {0} \left(\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} \left(s ^ {\prime}, a _ {1}\right)\right) - \max  _ {a _ {3}} f \left(\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} \left(s ^ {\prime}, a _ {3}\right)\right)\right) \right. \\ \left. - \sum_ {s ^ {\prime}} p (s ^ {\prime} | s, 0) \left(\max  _ {a _ {2}} f _ {0} (\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} (s ^ {\prime}, a _ {2})) - \max  _ {a _ {4}} f (\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} (s ^ {\prime}, a _ {4})) \right\| ^ {2} \right. \\ \leq 2 \| \max  _ {(s, a)} f _ {0} (\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} (s, a)) - f (\boldsymbol {\theta} _ {k}; \boldsymbol {\phi} (s, a)) \| ^ {2} \\ \leq 2 \mathcal {O} \left(\frac {c _ {1} ^ {3} \left(\left\| \boldsymbol {\theta} _ {0} \right\| + \left| \lambda_ {0} \right| + 1\right) ^ {3}}{m ^ {1 / 2}}\right), \tag {33} \\ \end{array}
$$
where the last inequality is due to Lemma 10. Substituting (32) and (33) back to (30) yields the final results.
D Proof of the Theorem 2
To prove Theorem 2, we need the following three key lemmas about the error terms defined in (19).
Lemma 7. Let  $\{\pmb {\theta}_k,\lambda_k\}$  be generated by (9). Then under Lemmas 1-4, for any  $k\geq \tau$  , we have
$$
\begin{array}{l} \mathbb {E} \left[ \left\| \tilde {\boldsymbol {\theta}} _ {k + 1} \right\| ^ {2} | \mathcal {F} _ {k - \tau} \right] \leq (1 + 1 5 0 \alpha_ {k} ^ {2} + \eta_ {k} / \alpha_ {k} - 2 \alpha_ {k} \mu_ {1}) \mathbb {E} \left[ \left\| \hat {\boldsymbol {\theta}} _ {k} \right\| ^ {2} | \mathcal {F} _ {k - \tau} \right] + 6 \alpha_ {k} ^ {2} \mathbb {E} \left[ \left\| \hat {\lambda} _ {k} \right\| ^ {2} | \mathcal {F} _ {k - \tau} \right] \\ + \frac {\alpha_ {k} ^ {3}}{\eta_ {k}} \mathcal {O} \left(c _ {1} ^ {3} \left(\| \boldsymbol {\theta} _ {0} \| + | \lambda_ {0} | + 1\right) ^ {3} \cdot m ^ {- 1 / 2}\right). \tag {34} \\ \end{array}
$$
Proof. According to (19), we have  $\hat{\pmb{\theta}}_{k + 1} \coloneqq \pmb{\theta}_{k + 1} - \pmb{\theta}_0^* = \hat{\pmb{\theta}}_k + \alpha_k h(X_k, \pmb{\theta}_k, \lambda_k)$ , which leads to
$$
\begin{array}{l} \left\| \hat {\boldsymbol {\theta}} _ {k + 1} \right\| ^ {2} = \left\| \hat {\boldsymbol {\theta}} _ {k} \right\| ^ {2} + 2 \alpha_ {k} \hat {\boldsymbol {\theta}} _ {k} ^ {\intercal} h (X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}) + \left\| \alpha_ {k} h (X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}) \right\| ^ {2} \\ = \left\| \hat {\boldsymbol {\theta}} _ {k} \right\| ^ {2} + 2 \alpha_ {k} \hat {\boldsymbol {\theta}} _ {k} ^ {\intercal} \left(h \left(X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}\right) - h _ {0} \left(X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}\right)\right) + 2 \alpha_ {k} \hat {\boldsymbol {\theta}} _ {k} ^ {\intercal} h _ {0} \left(X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}\right) \\ + \alpha_ {k} ^ {2} \| h (X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}) - h _ {0} (X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}) + h _ {0} (X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}) \| ^ {2} \\ \leq \left\| \hat {\boldsymbol {\theta}} _ {k} \right\| ^ {2} + 2 \alpha_ {k} \hat {\boldsymbol {\theta}} _ {k} ^ {\intercal} (h (X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}) - h _ {0} (X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k})) + 2 \alpha_ {k} \hat {\boldsymbol {\theta}} _ {k} ^ {\intercal} h _ {0} (X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}) \\ + 2 \alpha_ {k} ^ {2} \| h \left(X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}\right) - h _ {0} \left(X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}\right) \| ^ {2} + 2 \alpha_ {k} ^ {2} \| h _ {0} \left(X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}\right) \| ^ {2}. \tag {35} \\ \end{array}
$$
The above inequality holds due to the fact that  $\| \pmb{x} + \pmb{y}\|^2 \leq 2\| \pmb{x}\|^2 + 2\| \pmb{y}\|^2$ . Taking expectations of  $\|\hat{\pmb{\theta}}_{k+1}\|^2$  w.r.t  $\mathcal{F}_{k-\tau}$  yields
$$
\begin{array}{l} \mathbb {E} \left[ \| \hat {\boldsymbol {\theta}} _ {k + 1} \| ^ {2} | \mathcal {F} _ {k - \tau} \right] \leq \mathbb {E} \left[ \left\| \hat {\boldsymbol {\theta}} _ {k} \right\| ^ {2} | \mathcal {F} _ {k - \tau} \right] + 2 \alpha_ {k} \mathbb {E} \left[ \hat {\boldsymbol {\theta}} _ {k} ^ {\intercal} h _ {0} \left(X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}\right) | \mathcal {F} _ {k - \tau} \right] \\ + \underbrace {2 \alpha_ {k} ^ {2} \mathbb {E} \left[ \left\| h _ {0} \left(X _ {k} , \boldsymbol {\theta} _ {k} , \lambda_ {k}\right) \right\| ^ {2} \mid \mathcal {F} _ {k - \tau} \right]} _ {\text {T e r m} _ {1}} \\ + \underbrace {2 \alpha_ {k} \mathbb {E} \left[ \boldsymbol {\theta} _ {k} ^ {\mathsf {T}} \left(h \left(X _ {k} , \boldsymbol {\theta} _ {k} , \lambda_ {k}\right) - h _ {0} \left(X _ {k} , \boldsymbol {\theta} _ {k} , \lambda_ {k}\right)\right) \mid \mathcal {F} _ {k - \tau} \right]} _ {\text {T e r m} _ {2}} \\ + \underbrace {2 \alpha_ {k} ^ {2} \mathbb {E} \left[ \left\| h (X _ {k} , \boldsymbol {\theta} _ {k} , \lambda_ {k}) - h _ {0} (X _ {k} , \boldsymbol {\theta} _ {k} , \lambda_ {k}) \right\| ^ {2} | \mathcal {F} _ {k - \tau} \right]} _ {\text {T e r m} _ {3}} \\ \end{array}
$$
$$
\leq \mathbb {E} \left[ \left\| \hat {\boldsymbol {\theta}} _ {k} \right\| ^ {2} \mid \mathcal {F} _ {k - \tau} \right] - 2 \alpha_ {k} \mu_ {1} \mathbb {E} \left[ \left\| \tilde {\boldsymbol {\theta}} _ {k} \right\| ^ {2} \mid \mathcal {F} _ {k - \tau} \right] + \operatorname {T e r m} _ {1} + \operatorname {T e r m} _ {2} + \operatorname {T e r m} _ {3}, \tag {36}
$$
where the last inequality is due to Lemma 4. Next, we bound each individual term.  $\mathrm{Term}_1$  is bounded as
$$
\mathbf {T e r m} _ {1} = 2 \alpha_ {k} ^ {2} \mathbb {E} \left[ \left\| h _ {0} \left(X _ {k}, \boldsymbol {\theta} _ {k}, \lambda_ {k}\right) \right\| ^ {2} \mid \mathcal {F} _ {k - \tau} \right]
$$