SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
s _ {t - \tau} \xrightarrow {\theta_ {t - \tau}} a _ {t - \tau} \xrightarrow {\mathcal {P}} s _ {t - \tau + 1} \xrightarrow {\theta_ {t - \tau + 1}} \widetilde {a} _ {t - \tau + 1} \xrightarrow {\mathcal {P}} \widetilde {s} _ {t - \tau + 2} \xrightarrow {\theta_ {t - \tau + 2}} \widetilde {a} _ {t - \tau + 2} \dots \xrightarrow {\mathcal {P}} \widetilde {s} _ {t} \xrightarrow {\theta_ {t}} \widetilde {a} _ {t} \xrightarrow {\mathcal {P}} \widetilde {s} _ {t + 1}. \tag {11}
$$
In the sequel, we denote by  $\widetilde{O}_t\coloneqq (\widetilde{s}_t,\widetilde{a}_t,\widetilde{s}_{t + 1})$  the tuple generated from the auxiliary Markov chain in (10) while  $O_{t}\coloneqq (s_{t},a_{t},s_{t + 1})$  denotes the tuple generated from the original Markov chain in (11).
We define the following functions, which will benefit to decompose the errors and simplify the presentation.
$$
\Delta g (O, \eta , \boldsymbol {\theta}) := [ J (\boldsymbol {\theta}) - \eta ] \phi (s),
$$
$$
g (O, \omega , \boldsymbol {\theta}) := [ r (s, a) - J (\boldsymbol {\theta}) + (\phi (s ^ {\prime}) - \phi (s)) ^ {\top} \omega ] \phi (s),
$$
$$
\bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta}) := \mathbb {E} _ {(s, a, s ^ {\prime}) \sim (\mu_ {\boldsymbol {\theta}}, \pi_ {\boldsymbol {\theta}}, \mathcal {P})} \left[ \left[ r (s, a) - J (\boldsymbol {\theta}) + \left(\phi \left(s ^ {\prime}\right) - \phi (s)\right) ^ {\top} \boldsymbol {\omega} \right] \phi (s) \right], \tag {12}
$$
$$
\Delta h (O, \eta , \omega , \boldsymbol {\theta}) := (J (\boldsymbol {\theta}) - \eta + (\phi (s ^ {\prime}) - \phi (s)) ^ {\top} (\omega - \omega^ {*} (\boldsymbol {\theta})) \nabla \log \pi_ {\boldsymbol {\theta}} (a | s),
$$
$$
\Delta h ^ {\prime} (O, \boldsymbol {\theta}) := ((\phi (s ^ {\prime}) \omega^ {*} (\boldsymbol {\theta}) - V _ {\boldsymbol {\theta}} (s ^ {\prime})) - (\phi (s) ^ {\top} \omega^ {*} (\boldsymbol {\theta}) - V _ {\boldsymbol {\theta}} (s))) \nabla \log \pi_ {\boldsymbol {\theta}} (a | s),
$$
$$
h (O, \boldsymbol {\theta}) := \left(r (s, a) - J (\boldsymbol {\theta}) + \phi \left(s ^ {\prime}\right) ^ {\top} \boldsymbol {\omega} ^ {*} (\boldsymbol {\theta}) - \phi (s) ^ {\top} \boldsymbol {\omega} ^ {*} (\boldsymbol {\theta})\right) \nabla \log \pi_ {\boldsymbol {\theta}} (a | s).
$$
We also define the following functions, which characterize the Markovian noise.
$$
\Phi (O, \eta , \boldsymbol {\theta}) := (\eta - J (\boldsymbol {\theta})) (r (s, a) - J (\boldsymbol {\theta})),
$$
$$
\begin{array}{l} \Psi (O, \omega , \boldsymbol {\theta}) := \langle \omega - \omega_ {\boldsymbol {\theta}} ^ {*}, g (O, \omega , \boldsymbol {\theta}) - \bar {g} (\omega , \boldsymbol {\theta}) \rangle , \\ \partial (O, O ^ {\prime}, \boldsymbol {\theta}) := \left\langle \nabla I (\boldsymbol {\theta}), \mathbb {E} _ {t} [ h (O ^ {\prime}, \boldsymbol {\theta}) ] - h (O, \boldsymbol {\theta}) \right\rangle \end{array} \tag {13}
$$
$$
\Theta (O, O ^ {\prime}, \boldsymbol {\theta}) := \left\langle \nabla J (\boldsymbol {\theta}), \mathbb {E} _ {O ^ {\prime}} \left[ h \left(O ^ {\prime}, \boldsymbol {\theta}\right) \right] - h (O, \boldsymbol {\theta}) \right\rangle , \tag {13}
$$
$$
\Xi (O, \omega , \boldsymbol {\theta}) := \left\langle \omega - \omega_ {\boldsymbol {\theta}} ^ {*}, \left(\nabla \omega_ {\boldsymbol {\theta}} ^ {*}\right) ^ {\top} \left(\mathbb {E} _ {O ^ {\prime}} [ h (O ^ {\prime}, \boldsymbol {\theta}) ] - h (O, \boldsymbol {\theta})\right) \right\rangle ,
$$
where  $O'$  is a shorthand for an independent sample from stationary distribution  $s \sim \mu_{\pmb{\theta}}$ ,  $a \sim \pi_{\pmb{\theta}}$ ,  $s' \sim \mathcal{P}$ . Define  $U_{\delta} \coloneqq 2U_r + 2U_\omega$  so that we have  $|\delta_t| \leq U_{\delta}$ , where  $\delta_t$  comes from Line 6 in Algorithm 1. Note that from Assumption 3.3, we have  $\| \delta \nabla \log \pi_{\pmb{\theta}} \| \leq G \coloneqq U_{\delta} B$ .
B Preliminary Lemmas
Lemma B.1 ([31], Lemma C.4). For any  $\theta_{1},\theta_{2}$  , we have
$$
| J (\boldsymbol {\theta} _ {1}) - J (\boldsymbol {\theta} _ {2}) | \leq L _ {J} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|,
$$
where  $L_{J} = 2U_{r}|\mathcal{A}|L_{\pi}(1 + \lceil \log_{\rho}m^{-1}\rceil +\frac{1}{1 - \rho})$
Lemma B.2 ([38], Lemma 3.2). For the performance function  $J(\pmb{\theta})$ , there exists a constant  $L_{J'} > 0$  such that for all  $\pmb{\theta}_1, \pmb{\theta}_2 \in \mathbb{R}^d$ , it holds that
$$
\left\| \nabla J \left(\boldsymbol {\theta} _ {1}\right) - \nabla J \left(\boldsymbol {\theta} _ {2}\right) \right\| \leq L _ {J ^ {\prime}} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|, \tag {14}
$$
which further implies
$$
J \left(\boldsymbol {\theta} _ {2}\right) \geq J \left(\boldsymbol {\theta} _ {1}\right) + \left\langle \nabla J \left(\boldsymbol {\theta} _ {1}\right), \boldsymbol {\theta} _ {2} - \boldsymbol {\theta} _ {1} \right\rangle - \frac {L _ {J ^ {\prime}}}{2} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \| ^ {2}, \tag {15}
$$
$$
J \left(\boldsymbol {\theta} _ {2}\right) \leq J \left(\boldsymbol {\theta} _ {1}\right) + \left\langle \nabla J \left(\boldsymbol {\theta} _ {1}\right), \boldsymbol {\theta} _ {2} - \boldsymbol {\theta} _ {1} \right\rangle + \frac {L _ {J ^ {\prime}}}{2} \left\| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \right\| ^ {2}. \tag {16}
$$
Lemma B.3 ([31], Proposition 4.4). There exists a constant  $L_{*} > 0$  such that
$$
\left\| \boldsymbol {\omega} ^ {*} (\boldsymbol {\theta} _ {1}) - \boldsymbol {\omega} ^ {*} (\boldsymbol {\theta} _ {2}) \right\| \leq L _ {*} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|, \forall \boldsymbol {\theta} _ {1}, \boldsymbol {\theta} _ {2} \in \mathbb {R} ^ {d},
$$
where  $L_{*} = (2\lambda^{-2}U_{r} + 3\lambda^{-1}U_{r})|\mathcal{A}|L_{\pi}(1 + \lceil \log_{\rho}m^{-1}\rceil +\frac{1}{1 - \rho}).$
Lemma B.4 ([8], Proposition 8). For any  $\theta_{1},\theta_{2}\in \mathbb{R}^{d}$ , we have
$$
\left\| \nabla \omega^ {*} \left(\boldsymbol {\theta} _ {1}\right) - \nabla \omega^ {*} \left(\boldsymbol {\theta} _ {2}\right) \right\| \leq L _ {s} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|,
$$
where  $L_{s}$  is a positive constant.
Lemma B.5 ([42],[31]). For any  $\theta_{1}$  and  $\theta_{2}$ , it holds that
$$
d _ {T V} (\mu_ {\pmb {\theta} _ {1}}, \mu_ {\pmb {\theta} _ {2}}) \leq | \mathcal {A} | (\lceil \log_ {\rho} m ^ {- 1} \rceil + \frac {1}{1 - \rho}) \| \pmb {\theta} _ {1} - \pmb {\theta} _ {2} \|,
$$
$$
d _ {T V} \left(\mu_ {\boldsymbol {\theta} _ {1}} \otimes \pi_ {\boldsymbol {\theta} _ {1}}, \mu_ {\boldsymbol {\theta} _ {2}} \otimes \pi_ {\boldsymbol {\theta} _ {2}}\right) \leq | \mathcal {A} | L _ {\pi} \left(1 + \lceil \log_ {\rho} m ^ {- 1} \rceil + \frac {1}{1 - \rho}\right) \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|,
$$
$$
d _ {T V} (\mu_ {\pmb {\theta} _ {1}} \otimes \pi_ {\pmb {\theta} _ {1}} \otimes \mathcal {P}, \mu_ {\pmb {\theta} _ {2}} \otimes \pi_ {\pmb {\theta} _ {2}} \otimes \mathcal {P}) \leq | \mathcal {A} | L _ {\pi} (1 + \lceil \log_ {\rho} m ^ {- 1} \rceil + \frac {1}{1 - \rho}) \| \pmb {\theta} _ {1} - \pmb {\theta} _ {2} \|.
$$
Lemma B.6 ([31], Lemma B.2). Given time indexes  $t$  and  $\tau$  such that  $t \geq \tau > 0$ , consider the auxiliary Markov chain in (10). Conditioning on  $s_{t - \tau + 1}$  and  $\theta_{t - \tau}$ , we have
$$
d _ {T V} \left(\mathbb {P} \left(s _ {t + 1} \in \cdot\right), \mathbb {P} \left(\widetilde {s} _ {t + 1} \in \cdot\right)\right) \leq d _ {T V} \left(\mathbb {P} \left(O _ {t} \in \cdot\right), \mathbb {P} \left(\widetilde {O} _ {t} \in \cdot\right)\right),
$$
$$
d _ {T V} (\mathbb {P} (O _ {t} \in \cdot), \mathbb {P} (\widetilde {O} _ {t} \in \cdot)) = d _ {T V} (\mathbb {P} ((s _ {t}, a _ {t}) \in \cdot), \mathbb {P} ((\widetilde {s} _ {t}, \widetilde {a} _ {t}) \in \cdot)),
$$
$$
d _ {T V} \left(\mathbb {P} \left(\left(s _ {t}, a _ {t}\right) \in \cdot\right), \mathbb {P} \left(\left(\widetilde {s} _ {t}, \widetilde {a} _ {t}\right) \in \cdot\right)\right) \leq d _ {T V} \left(\mathbb {P} \left(s _ {t} \in \cdot\right), \mathbb {P} \left(\widetilde {s} _ {t} \in \cdot\right)\right) + \frac {1}{2} | \mathcal {A} | \mathbb {E} [ \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \| ].
$$
C Proof of Main Theorem
C.1 Step 1: Reward estimation error analysis
In this subsection, we will establish an implicit bound for estimator.
Lemma C.1. From any  $t \geq \tau > 0$ , we have
$$
\begin{array}{l} \mathbb {E} \left[ \Phi \left(O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t}\right) \right] \leq 4 U _ {r} L _ {J} \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \| + 2 U _ {r} | \eta_ {t} - \eta_ {t - \tau} | \\ + 2 U _ {r} ^ {2} | \mathcal {A} | L _ {\pi} \sum_ {i = t - \tau} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {i} - \boldsymbol {\theta} _ {t - \tau} \| + 4 U _ {r} ^ {2} m \rho^ {\tau - 1}. \\ \end{array}
$$
Theorem C.2. Choose  $\alpha_{t} = \frac{c}{\sqrt{T}}, \beta_{t} = \gamma_{t} = \frac{1}{\sqrt{T}}$ , we have
$$
Y _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + c G \sqrt {Y _ {T} G _ {T}}. \tag {17}
$$
Proof. From the update rule of reward estimator in Line 7 of Algorithm 1, we have
$$
\eta_ {t + 1} - J (\boldsymbol {\theta} _ {t + 1}) = \eta_ {t} - J (\boldsymbol {\theta} _ {t}) + J (\boldsymbol {\theta} _ {t}) - J (\boldsymbol {\theta} _ {t + 1}) + \gamma_ {t} (r _ {t} - \eta_ {t})
$$
$$
\begin{array}{l} y _ {t + 1} ^ {2} = \left(y _ {t} + J (\boldsymbol {\theta} _ {t}) - J (\boldsymbol {\theta} _ {t + 1}) + \gamma_ {t} \left(r _ {t} - \eta_ {t}\right)\right) ^ {2} \\ \leq y _ {t} ^ {2} + 2 y _ {t} \left(J \left(\boldsymbol {\theta} _ {t}\right) - J \left(\boldsymbol {\theta} _ {t + 1}\right)\right) + 2 \gamma_ {t} y _ {t} \left(r _ {t} - \eta_ {t}\right) \\ + 2 \left(J \left(\boldsymbol {\theta} _ {t}\right) - J \left(\boldsymbol {\theta} _ {t + 1}\right)\right) ^ {2} + 2 \gamma_ {t} ^ {2} \left(r _ {t} - \eta_ {t}\right) ^ {2} \\ = \left(1 - 2 \gamma_ {t}\right) y _ {t} ^ {2} + 2 \gamma_ {t} y _ {t} \left(r _ {t} - J \left(\boldsymbol {\theta} _ {t}\right)\right) + 2 y _ {t} \left(J \left(\boldsymbol {\theta} _ {t}\right) - J \left(\boldsymbol {\theta} _ {t + 1}\right)\right) \\ + 2 \left(J \left(\boldsymbol {\theta} _ {t}\right) - J \left(\boldsymbol {\theta} _ {t + 1}\right)\right) ^ {2} + 2 \gamma_ {t} ^ {2} \left(r _ {t} - \eta_ {t}\right) ^ {2}. \\ \end{array}
$$
Taking expectation up to  $s_{t + 1}$  (the whole trajectory), rearranging and summing from  $\tau_T$  to  $T - 1$ , we have
$$
\begin{array}{l} \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} [ y _ {t} ^ {2} ] \leq \underbrace {\sum_ {t = \tau_ {T}} ^ {T} \frac {1}{2 \gamma_ {t}} \mathbb {E} (y _ {t} ^ {2} - y _ {t + 1} ^ {2})} _ {I _ {1}} + \underbrace {\sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} [ y _ {t} (r _ {t} - J (\boldsymbol {\theta} _ {t})) ]} _ {I _ {2}} + \underbrace {\sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\gamma_ {t}} \mathbb {E} [ y _ {t} (J (\boldsymbol {\theta} _ {t}) - J (\boldsymbol {\theta} _ {t + 1}) ]} _ {I _ {3}} \\ + \underbrace {\sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\gamma_ {t}} \mathbb {E} [ (J (\boldsymbol {\theta} _ {t}) - J (\boldsymbol {\theta} _ {t + 1})) ^ {2} ]} _ {I _ {4}} + \underbrace {\sum_ {t = \tau_ {T}} ^ {T - 1} \gamma_ {t} \mathbb {E} [ (r _ {t} - \eta_ {t}) ^ {2} ]} _ {I _ {5}}. \\ \end{array}
$$
For term  $I_{1}$ , from Abel summation by parts, we have
$$
\begin{array}{l} I _ {1} = \sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{2 \gamma_ {t}} (y _ {t} ^ {2} - y _ {t + 1} ^ {2}) \\ = \sum_ {t = \tau T + 1} ^ {T - 1} y _ {t} ^ {2} (\frac {1}{2 \gamma_ {t}} - \frac {1}{2 \gamma_ {t - 1}}) + \frac {1}{2 \gamma_ {\tau_ {t}}} y _ {\tau_ {t}} ^ {2} - \frac {1}{\gamma_ {T - 1}} y _ {T} ^ {2} \\ \leq \frac {2 U _ {r} ^ {2}}{\gamma_ {T - 1}} \\ = 2 U _ {r} ^ {2} \sqrt {T}. \\ \end{array}
$$
For term  $I_{2}$  , from Lemma C.1, we have
$$
\begin{array}{l} \mathbb {E} \left[ y _ {t} \left(r _ {t} - J (\boldsymbol {\theta} _ {t})\right) \right] \leq 4 U _ {r} L _ {J} \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \| + 2 U _ {r} | \eta_ {t} - \eta_ {t - \tau} | \\ + 2 U _ {r} ^ {2} | \mathcal {A} | L _ {\pi} \sum_ {i = t - \tau} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {i} - \boldsymbol {\theta} _ {t - \tau} \| + 4 U _ {r} ^ {2} m \rho^ {\tau - 1} \\ \leq 4 U _ {r} L _ {J} G \tau \alpha_ {t - \tau} + 4 U _ {r} ^ {2} \tau \gamma_ {t - \tau} + 2 U _ {r} ^ {2} | A | L _ {\pi} \tau (\tau + 1) G \alpha_ {t - \tau} + 4 U _ {r} ^ {2} m \rho^ {- 1} \\ \leq \left(4 U _ {r} L _ {J} G \tau + 2 U _ {r} ^ {2} | A | L _ {\pi} G \tau (\tau + 1)\right) \alpha_ {t - \tau} + 4 U _ {r} ^ {2} \tau \gamma_ {t - \tau} + 4 U _ {r} ^ {2} m \rho^ {\tau - 1}. \\ \end{array}
$$
Choose  $\tau = \tau_{T}$ , we have
$$
\begin{array}{l} I _ {2} = \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \left[ y _ {t} \left(r _ {t} - J (\boldsymbol {\theta} _ {t})\right) \right] \\ \leq \left(4 U _ {r} L _ {J} G \tau_ {T} + 2 U _ {r} ^ {2} | \mathcal {A} | L _ {\pi} G \tau_ {T} (\tau_ {T} + 1)\right) \sum_ {t = \tau_ {T}} ^ {T - 1} \alpha_ {t} \\ + 4 U _ {r} ^ {2} \tau_ {T} \sum_ {t = \tau_ {T}} ^ {T - - 1} \gamma_ {t} + 4 U _ {r} ^ {2} \sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\sqrt {T}} \\ = \left(4 U _ {r} L _ {J} G \tau_ {T} + 2 U _ {r} ^ {2} | \mathcal {A} | L _ {\pi} G \tau_ {T} (\tau_ {T} + 1) + 4 U _ {r} ^ {2} \tau_ {T} + 4 U _ {r} ^ {2}\right) \frac {T - \tau_ {T}}{\sqrt {T}}. \\ \end{array}
$$
For  $I_{3}$  ,if  $y_{t} > 0$  ,from (15),we have
$$
\begin{array}{l} y _ {t} \left(J \left(\boldsymbol {\theta} _ {t}\right) - J \left(\boldsymbol {\theta} _ {t + 1}\right)\right) \leq y _ {t} \left(\frac {L _ {J ^ {\prime}}}{2} \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \right\| ^ {2} + \left\langle \nabla J \left(\boldsymbol {\theta} _ {t}\right), \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \right\rangle\right) \\ \leq L _ {J ^ {\prime}} U _ {r} \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \right\| ^ {2} + | y _ {t} | \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \right\| \left\| \nabla J (\boldsymbol {\theta} _ {t}) \right\|. \\ \end{array}
$$
If  $y_{t}\leq 0$  , from (16), we have
$$
\begin{array}{l} y _ {t} \left(J \left(\boldsymbol {\theta} _ {t}\right) - J \left(\boldsymbol {\theta} _ {t + 1}\right)\right) \leq y _ {t} \left(- \frac {L _ {J ^ {\prime}}}{2} \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \right\| ^ {2} + \left\langle \nabla J \left(\boldsymbol {\theta} _ {t}\right), \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \right\rangle\right) \\ \leq L _ {J ^ {\prime}} U _ {r} \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \right\| ^ {2} + | y _ {t} | \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \| \| \nabla J (\boldsymbol {\theta} _ {t}) \|. \\ \end{array}
$$
Overall, we get
$$
\begin{array}{l} I _ {3} = \sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\gamma_ {t}} \mathbb {E} \left[ y _ {t} \left(J \left(\boldsymbol {\theta} _ {t}\right) - J \left(\boldsymbol {\theta} _ {t + 1}\right)\right) \right] \\ \leq \sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\gamma_ {t}} \mathbb {E} \left[ L _ {J ^ {\prime}} U _ {r} \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \| ^ {2} + | y _ {t} | \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \| \| \nabla J (\boldsymbol {\theta} _ {t}) \| \right] \\ \leq \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \left[ c L _ {J ^ {\prime}} U _ {r} G ^ {2} \alpha_ {t} + c G | y _ {t} | \| \nabla J (\boldsymbol {\theta} _ {t}) \| \right] \\ \leq c L _ {J ^ {\prime}} U _ {r} G ^ {2} \frac {T - \tau_ {T}}{\sqrt {T}} + c G \left(\sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} y _ {t} ^ {2}\right) ^ {\frac {1}{2}} \left(\sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2}\right) ^ {\frac {1}{2}}. \\ \end{array}
$$
For term  $I_4$ , we have
$$
\begin{array}{l} I _ {4} = \sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\gamma_ {t}} \mathbb {E} \left[ \left(J (\boldsymbol {\theta} _ {t}) - J (\boldsymbol {\theta} _ {t + 1})\right) ^ {2} \right] \\ \leq \sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\gamma_ {t}} L _ {J} ^ {2} \mathbb {E} \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t + 1} \| ^ {2} \\ \leq \sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\gamma_ {t}} L _ {J} ^ {2} G ^ {2} \alpha_ {t} ^ {2} \\ = L _ {J} ^ {2} G ^ {2} c ^ {2} \frac {T - \tau_ {T}}{\sqrt {T}}. \\ \end{array}
$$
For term  $I_{5}$ , we have
$$
\begin{array}{l} I _ {5} = \sum_ {t = \tau_ {T}} ^ {T - 1} \gamma_ {t} \mathbb {E} \left[ \left(r _ {t} - J (\boldsymbol {\theta} _ {t})\right) ^ {2} \right] \\ \leq \sum_ {t = \tau_ {T}} ^ {T - 1} 4 U _ {r} ^ {2} \gamma_ {t} \\ = 4 U _ {r} ^ {2} \frac {T - \tau_ {T}}{\sqrt {T}}. \\ \end{array}
$$
$$
\begin{array}{l} \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \left[ y _ {t} ^ {2} \right] \leq \left(4 U _ {r} L _ {J} G \tau_ {T} + 2 U _ {r} ^ {2} | \mathcal {A} | L _ {\pi} G \tau_ {T} (\tau_ {T} + 1) \right. \\ + 4 U _ {r} ^ {2} \left(\tau_ {T} + 2\right) + c ^ {2} G ^ {2} \left(L _ {J ^ {\prime}} U _ {r} + L _ {J} ^ {2}\right)) \frac {T - \tau_ {T}}{\sqrt {T}} \\ + 2 U _ {r} ^ {2} \sqrt {T} + c G \left(\sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} y _ {t} ^ {2}\right) ^ {\frac {1}{2}} \left(\sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2}\right) ^ {\frac {1}{2}}. \\ \end{array}
$$
Since  $\tau_T = \mathcal{O}(\log T)$ , we have  $\frac{\sqrt{T}}{T - \tau_T} \leq \frac{2}{\sqrt{T}}$  for large  $T$ . Then we get
$$
\begin{array}{l} \frac {1}{T - \tau_ {T}} \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} [ y _ {t} ^ {2} ] \\ \leq \left(4 U _ {r} L _ {J} G \tau_ {T} + 2 U _ {r} ^ {2} | \mathcal {A} | L _ {\pi} G \tau_ {T} (\tau_ {T} + 1) \right. \\ + 4 U _ {r} ^ {2} \left(\tau_ {T} + 3\right) + c ^ {2} G ^ {2} \left(L _ {J ^ {\prime}} U _ {r} + L _ {J} ^ {2}\right)) \frac {1}{\sqrt {T}} \\ + c G \left(\frac {1}{T - \tau_ {T}} \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} y _ {t} ^ {2}\right) ^ {\frac {1}{2}} \left(\frac {1}{T - \tau_ {T}} \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2}\right) ^ {\frac {1}{2}} \\ = \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + c G \left(\frac {1}{T - \tau_ {T}} \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} y _ {t} ^ {2}\right) ^ {\frac {1}{2}} \left(\frac {1}{T - \tau_ {T}} \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2}\right) ^ {\frac {1}{2}}. \\ \end{array}
$$
Thus we finish the proof.
C.2 Step 2: Critic error analysis
In this subsection, we will establish an implicit upper bound for critic.
Lemma C.3. For any  $t \geq \tau > 0$ , we have
$$
\mathbb {E} \left[ \Psi \left(O _ {t}, \omega_ {t}, \boldsymbol {\theta} _ {t}\right) \right] \leq C _ {1} \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \right\| + 6 U _ {\delta} \left\| \boldsymbol {\omega} _ {t} - \boldsymbol {\omega} _ {t - \tau} \right\| + U _ {\delta} ^ {2} | \mathcal {A} | L _ {\pi} G \tau (\tau + 1) \alpha_ {t - \tau} + 2 U _ {\delta} ^ {2} m \rho^ {\tau - 1},
$$
where  $C_1 = 2U_\delta^2 |\mathcal{A}|L_\pi (1 + \lceil \log_\rho m^{-1}\rceil +\frac{1}{1 - \rho}) + 2U_\delta L_J + 2U_\delta L_*$
Lemma C.4. Given the definition of  $\Xi (O_t,\pmb {\omega}_t,\pmb {\theta}_t)$  , for any  $t\geq \tau >0$  we have
$$
\begin{array}{l} \mathbb {E} \left[ \Xi \left(O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}\right) \right] \leq C _ {2} \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \right\| + 2 U _ {\delta} B \| \boldsymbol {\omega} _ {t} - \boldsymbol {\omega} _ {t - \tau} \| \\ + 2 U _ {\delta} ^ {2} B | \mathcal {A} | L _ {\pi} G \tau (\tau + 1) \alpha_ {t - \tau} + 4 U _ {\delta} ^ {2} B m \rho^ {\tau - 1}. \\ \end{array}
$$
where  $C_2 \coloneqq 3BU_{\delta}^2 |\mathcal{A}|L_{\pi}(1 + \lceil \log_{\rho}m^{-1}\rceil +\frac{1}{1 - \rho}) + 3U_{\delta}^2 L_l + 8U_{\delta}BL_*$
Theorem C.5. Choose  $\alpha_{t} = \frac{c}{\sqrt{T}},\beta_{t} = \gamma_{t} = \frac{1}{\sqrt{T}}$  we have
$$
Z _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\mathrm {a p p}}\right) + \frac {2}{\lambda} \sqrt {Y _ {T} Z _ {T}} + \frac {2 c B L _ {*}}{\lambda} \sqrt {Z _ {T} \left(2 Y _ {T} + 8 Z _ {T}\right)}. \tag {18}
$$
Proof. From the update rule of critic in Line 8 of Algorithm 1, we have
$$
\begin{array}{l} \left\| \omega_ {t + 1} - \omega_ {t + 1} ^ {*} \right\| = \left\| \Pi_ {U _ {\omega}} \left(\omega_ {t} + \beta_ {t} \delta_ {t} \phi (s _ {t})\right) - \omega_ {t + 1} ^ {*} \right\| \\ = \| \Pi_ {U _ {\omega}} \left(\omega_ {t} + \beta_ {t} \delta_ {t} \phi (s _ {t})\right) - \Pi_ {U _ {\omega}} \left(\omega_ {t + 1} ^ {*}\right) \| \\ \leq \left\| \omega_ {t} + \beta_ {t} \delta_ {t} \phi (s _ {t}) - \omega_ {t + 1} ^ {*} \right\| \\ = \| \boldsymbol {\omega} _ {t} + \beta_ {t} (g (O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) + \Delta g (O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t})) - \boldsymbol {\omega} _ {t + 1} ^ {*} \| \\ = \| \boldsymbol {\omega} _ {t} - \boldsymbol {\omega} _ {t} ^ {*} + \beta_ {t} (g (O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) + \Delta g (O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t})) + \boldsymbol {\omega} _ {t} ^ {*} - \boldsymbol {\omega} _ {t + 1} ^ {*} \|. \\ \end{array}
$$
$$
\begin{array}{l} \left\| \boldsymbol {z} _ {t + 1} \right\| ^ {2} = \left\| \boldsymbol {z} _ {t} + \beta_ {t} \left(g \left(O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}\right) + \Delta g \left(O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t}\right)\right) + \boldsymbol {\omega} _ {t} ^ {*} - \boldsymbol {\omega} _ {t + 1} ^ {*} \right\| ^ {2} \\ = \| \boldsymbol {z} _ {t} \| ^ {2} + 2 \beta_ {t} \langle \boldsymbol {z} _ {t}, g (O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) \rangle + 2 \beta_ {t} \langle \boldsymbol {z} _ {t}, \Delta g (O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t}) \rangle \\ + 2 \left\langle \boldsymbol {z} _ {t}, \boldsymbol {\omega} _ {t} ^ {*} - \boldsymbol {\omega} _ {t + 1} ^ {*} \right\rangle + \| \beta_ {t} (g (O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) + \Delta g (O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t})) + \boldsymbol {\omega} _ {t} ^ {*} - \boldsymbol {\omega} _ {t + 1} ^ {*} \| ^ {2} \\ = \| \boldsymbol {z} _ {t} \| ^ {2} + 2 \beta_ {t} \langle \boldsymbol {z} _ {t}, \bar {g} (\boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) \rangle + 2 \beta_ {t} \Psi (O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) + 2 \beta_ {t} \langle \boldsymbol {z} _ {t}, \Delta g (O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t}) \rangle \\ + 2 \left\langle z _ {t}, \omega_ {t} ^ {*} - \omega_ {t + 1} ^ {*} \right\rangle + \left\| \beta_ {t} \left(g \left(O _ {t}, \omega_ {t}, \boldsymbol {\theta} _ {t}\right) + \Delta g \left(O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t}\right)\right) + \omega_ {t} ^ {*} - \omega_ {t + 1} ^ {*} \right\| ^ {2} \\ \leq \left\| \boldsymbol {z} _ {t} \right\| ^ {2} + 2 \beta_ {t} \langle \boldsymbol {z} _ {t}, \bar {g} (\boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) \rangle + 2 \beta_ {t} \Psi (O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) + 2 \beta_ {t} \langle \boldsymbol {z} _ {t}, \Delta g (O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t}) \rangle \\ + 2 \left\langle \boldsymbol {z} _ {t}, \boldsymbol {\omega} _ {t} ^ {*} - \boldsymbol {\omega} _ {t + 1} ^ {*} \right\rangle + 2 U _ {\delta} ^ {2} \beta_ {t} ^ {2} + 2 \| \boldsymbol {\omega} _ {t} ^ {*} - \boldsymbol {\omega} _ {t + 1} ^ {*} \| ^ {2}. \\ \end{array}
$$
Note that we have