SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} J \left(\boldsymbol {\theta} _ {t + 1}\right) \geq J \left(\boldsymbol {\theta} _ {t}\right) + \left\langle \nabla J \left(\boldsymbol {\theta} _ {t}\right), \boldsymbol {\theta} _ {t + 1} - \boldsymbol {\theta} _ {t} \right\rangle - \frac {L _ {J ^ {\prime}}}{2} \left\| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \right\| ^ {2} \\ = J (\boldsymbol {\theta} _ {t}) + \left\langle \nabla J (\boldsymbol {\theta} _ {t}), \delta_ {t} \nabla \log \pi_ {\boldsymbol {\theta} _ {t}} (a _ {t} | s _ {t}) \right\rangle - \frac {L _ {J ^ {\prime}}}{2} \alpha_ {t} ^ {2} \| \delta_ {t} \nabla \log \pi_ {\boldsymbol {\theta} _ {t}} (a _ {t} | s _ {t}) \| ^ {2} \\ = J (\boldsymbol {\theta} _ {t}) + \alpha_ {t} \left\langle \nabla J (\boldsymbol {\theta} _ {t}), \Delta h \left(O _ {t}, \eta_ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}\right) \right\rangle + \alpha_ {t} \left\langle \nabla J (\boldsymbol {\theta} _ {t}), h \left(O _ {t}, \boldsymbol {\theta} _ {t}\right) \right\rangle \\ - \frac {L _ {J ^ {\prime}}}{2} \alpha_ {t} ^ {2} \| \delta_ {t} \nabla \log \pi_ {\boldsymbol {\theta} _ {t}} (a _ {t} | s _ {t}) \| ^ {2} \\ = J (\boldsymbol {\theta} _ {t}) + \alpha_ {t} \left\langle \nabla J (\boldsymbol {\theta} _ {t}), \Delta h \left(O _ {t}, \eta_ {t}, \omega_ {t}, \boldsymbol {\theta} _ {t}\right) \right\rangle - \alpha_ {t} \Theta \left(O _ {t}, \boldsymbol {\theta} _ {t}\right) \\ + \alpha_ {t} \left\langle \nabla J (\boldsymbol {\theta} _ {t}), \mathbb {E} _ {O ^ {\prime}} \left[ h (O ^ {\prime}, \boldsymbol {\theta} _ {t}) \right] \right\rangle - \frac {L _ {J ^ {\prime}}}{2} \alpha_ {t} ^ {2} \| \delta_ {t} \nabla \log \pi_ {\boldsymbol {\theta} _ {t}} \left(a _ {t} | s _ {t}\right) \| ^ {2} \\ = J (\boldsymbol {\theta} _ {t}) + \alpha_ {t} \left\langle \nabla J (\boldsymbol {\theta} _ {t}), \Delta h \left(O _ {t}, \eta_ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}\right) \right\rangle - \alpha_ {t} \Theta \left(O _ {t}, \boldsymbol {\theta} _ {t}\right) + \alpha_ {t} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2} \\ + \alpha_ {t} \langle \nabla J (\pmb {\theta} _ {t}), \mathbb {E} _ {O ^ {\prime}} [ \Delta h ^ {\prime} (O ^ {\prime}, \pmb {\theta} _ {t}) ] \rangle - \frac {L _ {J ^ {\prime}}}{2} \alpha_ {t} ^ {2} \| \delta_ {t} \nabla \log \pi_ {\pmb {\theta} _ {t}} (a _ {t} | s _ {t}) \| ^ {2}, \\ \end{array}
$$
where the last equality is due to the fact
$$
\mathbb {E} _ {O ^ {\prime}} \left[ h \left(O ^ {\prime}, \boldsymbol {\theta}\right) - \Delta h ^ {\prime} \left(O ^ {\prime}, \boldsymbol {\theta}\right) \right] = \mathbb {E} _ {O ^ {\prime}} \left[ \left(r (s, a) - J (\boldsymbol {\theta}) + V _ {\boldsymbol {\theta}} \left(s ^ {\prime}\right) - V _ {\boldsymbol {\theta}} (s)\right) \nabla \log \pi_ {\boldsymbol {\theta}} (a | s) \right] = \nabla J (\boldsymbol {\theta}).
$$
Rearranging the above inequality and taking expectation, we have
$$
\begin{array}{l} \mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2} \leq \frac {1}{\alpha_ {t}} (\mathbb {E} [ J (\boldsymbol {\theta} _ {t + 1}) - J (\boldsymbol {\theta} _ {t}) ]) - \mathbb {E} \langle \nabla J (\boldsymbol {\theta} _ {t}), \Delta h (O _ {t}, \eta_ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) \rangle + \mathbb {E} [ \Theta (O _ {t}, \boldsymbol {\theta} _ {t}) ] \\ - \mathbb {E} \langle \nabla J (\boldsymbol {\theta} _ {t}), \mathbb {E} _ {O ^ {\prime}} [ \Delta h ^ {\prime} (O ^ {\prime}, \boldsymbol {\theta} _ {t}) ] \rangle + \frac {L _ {J ^ {\prime}}}{2} \alpha_ {t} \mathbb {E} \| \delta_ {t} \nabla \log \pi_ {\boldsymbol {\theta} _ {t}} (a _ {t} | s _ {t}) \| ^ {2}. \\ \end{array}
$$
Note that from Cauchy-Schwartz inequality, we have
$$
- \mathbb {E} \langle \nabla J (\boldsymbol {\theta} _ {t}), \Delta h (O _ {t}, \eta_ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}) \rangle \leq B \sqrt {\mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2}} \sqrt {2 \mathbb {E} y _ {t} ^ {2} + 8 \mathbb {E} \| \boldsymbol {z} _ {t} \| ^ {2}}.
$$
From Lemma C.6 and choosing  $\tau = \tau_{T}$ , we have
$$
\begin{array}{l} \mathbb {E} \left[ \Theta \left(O _ {t}, \boldsymbol {\theta} _ {t}\right) \right] \leq D _ {1} \left(\tau_ {T} + 1\right) \sum_ {k = t - \tau_ {T} + 1} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} _ {k - 1} \| + D _ {2} m \rho^ {\tau_ {T} - 1} \\ \leq D _ {1} \left(\tau_ {T} + 1\right) G \sum_ {k = t - \tau_ {T}} ^ {t - 1} \alpha_ {k} + D _ {2} m \rho^ {\tau_ {T} - 1} \\ \leq G D _ {1} \left(\tau_ {T} + 1\right) ^ {2} \alpha_ {t - \tau_ {T}} + D _ {2} \frac {1}{\sqrt {T}}. \\ \end{array}
$$
It has been shown that
$$
\begin{array}{l} \mathbb {E} _ {O ^ {\prime}} \| \nabla h ^ {\prime} (O, \boldsymbol {\theta}) \| ^ {2} = \mathbb {E} _ {O ^ {\prime}} \| \left(\left(\phi \left(s ^ {\prime}\right) ^ {\top} \boldsymbol {\omega} ^ {*} - V _ {\boldsymbol {\theta}} \left(s ^ {\prime}\right)\right) - \left(\phi \left(s\right) ^ {\top} \boldsymbol {\omega} ^ {*} - V _ {\boldsymbol {\theta}} \left(s\right)\right)\right) \nabla \log \pi_ {\boldsymbol {\theta}} (a | s) \| ^ {2} \\ \leq \mathbb {E} _ {O ^ {\prime}} \left[ B ^ {2} \left(\left(\phi (s ^ {\prime}) ^ {\top} \omega^ {*} - V _ {\theta} (s ^ {\prime})\right) - \left(\phi (s) ^ {\top} \omega^ {*} - V _ {\theta} (s)\right)\right) ^ {2} \right] \\ \leq \mathbb {E} _ {O ^ {\prime}} \left[ 2 B ^ {2} \left(\phi \left(s ^ {\prime}\right) ^ {\top} \omega^ {*} - V _ {\boldsymbol {\theta}} \left(s ^ {\prime}\right)\right) ^ {2} + \left(\phi (s) ^ {\top} \omega^ {*} - V _ {\boldsymbol {\theta} (s)}\right) ^ {2} \right] \\ = 4 B ^ {2} \mathbb {E} _ {\mathcal {O} ^ {\prime}} \left[ \left(\phi (s) ^ {\top} \omega^ {*} - V _ {\theta} (s)\right) ^ {2} \right] \\ = 4 B ^ {2} \epsilon_ {\mathrm {a p p}} ^ {2}. \\ \end{array}
$$
$$
\begin{array}{l} - \left\langle \nabla J (\boldsymbol {\theta} _ {t}), \mathbb {E} _ {O ^ {\prime}} \left[ \Delta h ^ {\prime} \left(O ^ {\prime}, \boldsymbol {\theta} _ {t}\right) \right] \right\rangle \leq L _ {J} \sqrt {\left\| \mathbb {E} _ {O ^ {\prime}} \left[ \Delta h ^ {\prime} \left(O _ {t} , \boldsymbol {\theta} _ {t}\right) \right] \right\| ^ {2}} \\ \leq L _ {J} \sqrt {\mathbb {E} _ {O ^ {\prime}} \| \Delta h ^ {\prime} (O _ {t} , \boldsymbol {\theta} _ {t}) \| ^ {2}} \\ \leq 2 B L _ {J} \epsilon_ {\mathrm {a p p}}, \\ \end{array}
$$
where we use  $\| \nabla J(\pmb {\theta})\| \leq L_J$  which comes from Lemma B.1. Plugging the three terms yields
$$
\begin{array}{l} \mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2} \leq \frac {1}{\alpha_ {t}} \left(\mathbb {E} \left[ J (\boldsymbol {\theta} _ {t + 1}) \right] - \mathbb {E} \left[ J (\boldsymbol {\theta} _ {t}) \right]\right) + B \sqrt {\mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2}} \sqrt {2 \mathbb {E} y _ {t} ^ {2} + 8 \mathbb {E} \| \boldsymbol {z} _ {t} \| ^ {2}} \\ + 2 B L _ {J} \epsilon_ {\mathrm {a p p}} + G D _ {1} (\tau_ {T} + 1) ^ {2} \alpha_ {t - \tau_ {T}} + D _ {2} \frac {1}{\sqrt {T}} + \frac {L _ {J ^ {\prime}}}{2} G ^ {2} \alpha_ {t}. \\ \end{array}
$$
Summing over  $t$  from  $\tau_T$  to  $T - 1$  gives
$$
\begin{array}{l} \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2} \leq \underbrace {\sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\alpha_ {t}} (\mathbb {E} [ J (\boldsymbol {\theta} _ {t + 1}) - \mathbb {E} [ J (\boldsymbol {\theta} _ {t}) ])} _ {I _ {1}} + B \sum_ {t = \tau_ {T}} ^ {T - 1} \sqrt {\mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2}} \sqrt {2 \mathbb {E} y _ {t} ^ {2} + 8 \mathbb {E} \| \boldsymbol {z} _ {t} \| ^ {2}} \\ + \left(G D _ {1} \left(\tau_ {T} + 1\right) ^ {2} + D _ {2}\right) \frac {T - \tau_ {T}}{\sqrt {T}} + 2 B L _ {J} \epsilon_ {\text {a p p}} \left(T - \tau_ {T}\right). \\ \end{array}
$$
$$
\begin{array}{l} I _ {1} = \sum_ {t = \tau_ {T}} ^ {T - 1} \frac {1}{\alpha_ {t}} \left(\mathbb {E} \left[ J \left(\boldsymbol {\theta} _ {t + 1}\right) - \mathbb {E} \left[ J \left(\boldsymbol {\theta} _ {t}\right) \right]\right) \right. \\ = \sum_ {t = \tau_ {T} + 1} ^ {T - 1} \left(\frac {1}{\alpha_ {t - 1}} - \frac {1}{\alpha_ {t}}\right) \mathbb {E} [ J (\boldsymbol {\theta} _ {t}) ] - \mathbb {E} [ J (\boldsymbol {\theta} _ {\tau_ {T}}) ] \frac {1}{\alpha_ {\tau_ {T}}} + \frac {1}{\alpha_ {T - 1}} \mathbb {E} [ J (\boldsymbol {\theta} _ {T}) ] \\ \leq \sum_ {t = \tau_ {T} + 1} ^ {T - 1} \left(\frac {1}{\alpha_ {t}} - \frac {1}{\alpha_ {t - 1}}\right) U _ {r} + \frac {1}{\alpha_ {\tau_ {T}}} U _ {r} + \frac {1}{\alpha_ {T - 1}} U _ {r} \\ = \frac {2 U _ {r}}{\alpha_ {T - 1}} \\ = \frac {2 U _ {r}}{c} \sqrt {T}. \\ \end{array}
$$
$$
\begin{array}{l} \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2} \leq \frac {2 U _ {r}}{c} \sqrt {T} + (G D _ {1} (\tau_ {T} + 1) ^ {2} + D _ {2}) \frac {T - \tau_ {T}}{\sqrt {T}} + 2 B L _ {J} \epsilon_ {\mathrm {a p p}} (T - \tau_ {T}) \\ + B \sum_ {t = \tau_ {T}} ^ {T - 1} \sqrt {\mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2}} \sqrt {2 \mathbb {E} y _ {t} ^ {2} + 8 \mathbb {E} \| \boldsymbol {z} _ {t} \| ^ {2}} \\ \leq \frac {2 U _ {r}}{c} \sqrt {T} + \left(G D _ {1} \left(\tau_ {T} + 1\right) ^ {2} + D _ {2}\right) \frac {T - \tau_ {T}}{\sqrt {T}} + 2 B L _ {J} \epsilon_ {\mathrm {a p p}} \left(T - \tau_ {T}\right) \\ + B \left(\sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \| \nabla J (\boldsymbol {\theta} _ {t}) \| ^ {2}\right) ^ {\frac {1}{2}} \left(2 \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} y _ {t} ^ {2} + 8 \sum_ {t = \tau_ {T}} ^ {T - 1} \mathbb {E} \| \boldsymbol {z} _ {t} \| ^ {2}\right) ^ {\frac {1}{2}}. \\ \end{array}
$$
$$
\begin{array}{l} G _ {T} \leq \left(\frac {4 U _ {r}}{c} + G D _ {1} (\tau_ {T} + 1) ^ {2} + D _ {2}\right) \frac {1}{\sqrt {T}} + 2 B L _ {J} \epsilon_ {\mathrm {a p p}} + B \sqrt {G _ {T} (2 Y _ {T} + 8 Z _ {T})} \\ = \mathcal {O} (\frac {\log^ {2} T}{\sqrt {T}}) + \mathcal {O} (\epsilon_ {\mathrm {a p p}}) + B \sqrt {G _ {T} (2 Y _ {T} + 8 Z _ {T})}, \\ \end{array}
$$
C.4 Step 4: Interconnected iteration system analysis
In this subsection, we perform an interconnected iteration system analysis to prove Theorem 3.5.
Proof of Theorem 3.5.
Proof. Combining (17), (18), and (25), we have
$$
Y _ {T} \leq \mathcal {O} (\frac {\log^ {2} T}{\sqrt {T}}) + c G \sqrt {Y _ {T} G _ {T}},
$$
$$
G _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} (\epsilon_ {\mathrm {a p p}}) + B \sqrt {G _ {T} (2 Y _ {T} + 8 Z _ {T})}.
$$
$$
l _ {1} := c G,
$$
$$
l _ {2} := \frac {2}{\lambda},
$$
$$
l _ {3} := \frac {2 c B L _ {*}}{\lambda},
$$
$$
l _ {4} := B.
$$
$$
Y _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + l _ {1} \sqrt {Y _ {T} G _ {T}},
$$
$$
Z _ {T} \leq \mathcal {O} (\frac {\log^ {2} T}{\sqrt {T}}) + \mathcal {O} (\epsilon_ {\mathrm {a p p}}) + l _ {2} \sqrt {Y _ {T} Z _ {T}} + l _ {3} \sqrt {Z _ {T} (2 Y _ {T} + 8 Z _ {T})},
$$
$$
G _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\mathrm {a p p}}\right) + l _ {4} \sqrt {G _ {T} \left(2 Y _ {T} + 8 Z _ {T}\right)}.
$$
For  $G_{T}$ , we get
$$
G _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\mathrm {a p p}}\right) + \frac {1}{2} G _ {T} + l _ {4} ^ {2} \left(Y _ {T} + 4 Z _ {T}\right),
$$
$$
G _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\text {a p p}}\right) + 2 l _ {4} ^ {2} \left(Y _ {T} + 4 Z _ {T}\right). \tag {26}
$$
For  $Z_{T}$ , we have
$$
Z _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\mathrm {a p p}}\right) + \frac {1}{4} Z _ {T} + l _ {2} ^ {2} Y _ {T} + 4 l _ {3} Z _ {T} + \frac {l _ {3}}{2} Y _ {T}.
$$
If it satisfies  $4l_{3} \leq \frac{1}{4}$ , we further have
$$
Z _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\mathrm {a p p}}\right) + \left(2 l _ {2} ^ {2} + l _ {3}\right) Y _ {T}. \tag {27}
$$
For  $Y_{T}$  , we get
$$
Y _ {T} \leq \mathcal {O} (\frac {\log^ {2} T}{\sqrt {T}}) + \frac {l _ {1}}{2} (Y _ {T} + G _ {T}).
$$
Plugging (26) and (27) into the above inequality gives
$$
\begin{array}{l} Y _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\text {a p p}}\right) + \frac {l _ {1}}{2} \left(Y _ {T} + 2 l _ {4} ^ {2} Y _ {T} + 8 l _ {4} ^ {2} Z _ {T}\right) \\ \leq \mathcal {O} (\frac {\log^ {2} T}{\sqrt {T}}) + \mathcal {O} (\epsilon_ {\mathrm {a p p}}) + \frac {l _ {1}}{2} (Y _ {T} + 2 l _ {4} ^ {2} Y _ {T} + 8 l _ {4} ^ {2} (2 l _ {2} ^ {2} + l _ {3}) Y _ {T}) \\ = \mathcal {O} (\frac {\log^ {2} T}{\sqrt {T}}) + \mathcal {O} (\epsilon_ {\mathrm {a p p}}) + \frac {l _ {1}}{2} (1 + 2 l _ {4} ^ {2} + 8 l _ {4} ^ {2} (2 l _ {2} ^ {2} + l _ {3})) Y _ {T}. \\ \end{array}
$$
Therefore, if  $l_{1}(1 + 2l_{4}^{2} + 8l_{4}^{2}(2l_{2}^{2} + l_{3})\leq 1$  , we have
$$
Y _ {T} \leq \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\mathrm {a p p}}\right).
$$
Overall, we require
$$
4 l _ {3} \leq \frac {1}{4},
$$
$$
l _ {1} \left(1 + 2 l _ {4} ^ {2} + 8 l _ {4} ^ {2} \left(2 l _ {2} ^ {2} + l _ {3}\right)\right) \leq 1.
$$
According to the definition of  $l_{1}, l_{2}, l_{3}, l_{4}$ , we have
$$
\frac {8 c B L _ {*}}{\lambda} \leq \frac {1}{4},
$$
$$
c G (1 + 2 B ^ {2} + 8 B ^ {2} \left(\frac {8}{\lambda^ {2}} + \frac {2 c B L _ {*}}{\lambda}\right)) \leq 1.
$$
Thus we choose
$$
c = \min  \left\{\frac {\lambda}{3 2 B L _ {*}}, \frac {\lambda^ {2}}{G \left(\lambda^ {2} + 3 B ^ {2} \lambda^ {2} + 6 4 B ^ {2}\right)} \right\}, \tag {28}
$$
which satisfies the above two inequalities. Therefore, we have
$$
Y _ {T} = \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\mathrm {a p p}}\right),
$$
$$
Z _ {T} = \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\mathrm {a p p}}\right),
$$
$$
G _ {T} = \mathcal {O} \left(\frac {\log^ {2} T}{\sqrt {T}}\right) + \mathcal {O} \left(\epsilon_ {\mathrm {a p p}}\right).
$$
Thus we conclude our proof.
D Proof of Supporting Lemmas
The following three lemmas only deal with the Markovian noise, which are originally proved in [31] and updated in [30]. We include the proof with slight modifications for proving Theorem 3.5.
Proof of Lemma C.1.
Proof. We will divide the proof of this lemma into four steps.
Step 1: show that for any  $\theta_{1},\theta_{2},\eta ,O = (s,a,s^{\prime})$  , we have
$$
\left| \Phi (O, \eta , \boldsymbol {\theta} _ {1}) - \Phi (O, \eta , \boldsymbol {\theta} _ {2}) \right| \leq 4 U _ {r} L _ {J} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|. \tag {29}
$$
By the definition of  $\Phi (O,\eta ,\pmb {\theta})$  in (13), we have
$$
\begin{array}{l} \left| \Phi (O, \eta , \boldsymbol {\theta} _ {1}) - \Phi (O, \boldsymbol {\theta}, \boldsymbol {\theta} _ {2}) \right| = \left| (\eta - J (\boldsymbol {\theta} _ {1})) (r - J (\boldsymbol {\theta} _ {1})) - (\eta - J (\boldsymbol {\theta} _ {2})) (r - J (\boldsymbol {\theta} _ {2})) \right| \\ \leq | (\eta - J (\boldsymbol {\theta} _ {1})) (r - J (\boldsymbol {\theta} _ {1})) - (\eta - J (\boldsymbol {\theta} _ {1})) (r - J (\boldsymbol {\theta} _ {2})) | \\ + \left| \left(\eta - J \left(\boldsymbol {\theta} _ {1}\right)\right) \left(r - J \left(\boldsymbol {\theta} _ {2}\right)\right) - \left(\eta - J \left(\boldsymbol {\theta} _ {2}\right)\right) \left(r - J \left(\boldsymbol {\theta} _ {2}\right)\right) \right| \\ \leq 4 U _ {r} | J (\boldsymbol {\theta} _ {1}) - J (\boldsymbol {\theta} _ {2}) | \\ \leq 4 U _ {r} L _ {J} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|. \\ \end{array}
$$
Step 2: show that for any  $\theta, \eta_1, \eta_2, O$ , we have
$$
\left| \Phi (O, \eta_ {1}, \boldsymbol {\theta}) - \Phi (O, \eta_ {2}, \boldsymbol {\theta}) \leq 2 U _ {r} \right| \eta_ {1} - \eta_ {2} |. \tag {30}
$$
$$
\begin{array}{l} \left| \Phi (O, \eta_ {1}, \boldsymbol {\theta}) - \Phi (O, \eta_ {2}, \boldsymbol {\theta}) \right| = \left| (\eta_ {1} - J (\boldsymbol {\theta})) (r - J (\boldsymbol {\theta})) - (\eta_ {2} - J (\boldsymbol {\theta})) (r - J (\boldsymbol {\theta})) \right| \\ \leq 2 U _ {r} | \eta_ {1} - \eta_ {2} |. \\ \end{array}
$$
Step 3: show that for original tuple  $O_{t}$  and the auxiliary tuple  $\widetilde{O}_{t}$ , conditioned on  $s_{t - \tau -1}$  and  $\theta_{t - \tau}$  we have
$$
\left| \right. \mathbb {E} \left[ \right. \Phi \left(O _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) - \mathbb {E} \left[ \Phi \left(\widetilde {O} _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right)\right]\left. \right| \leq 2 U _ {r} ^ {2} | \mathcal {A} | L _ {\pi} \sum_ {k = t - \tau} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} _ {t - \tau} \|. \tag {31}
$$
$$
\mathbb {E} [ \Phi (O _ {t}, \eta_ {t - \tau}, \pmb {\theta} _ {t - \tau}) - \mathbb {E} [ \Phi (\widetilde {O} _ {t}, \eta_ {t - \tau}, \pmb {\theta} _ {t - \tau}) ] = (\eta_ {t - \tau} - J (\pmb {\theta} _ {t - \tau})) \mathbb {E} [ r (s _ {t}, a _ {t}) - r (\widetilde {s} _ {t}, \widetilde {a} _ {t}) ].
$$
By definition of total variation norm, we have
$$
\mathbb {E} \left[ r \left(s _ {t}, a _ {t}\right) - r \left(\widetilde {s} _ {t}, \widetilde {a} _ {t}\right) \right] \leq 2 U _ {r} d _ {T V} \left(\mathbb {P} \left(O _ {t} \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mathbb {P} \left(\widetilde {O} _ {t} \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right)\right). \tag {32}
$$
By Lemma B.6, we get
$$
\begin{array}{l} d _ {T V} (\mathbb {P} \left(O _ {t} \in \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mathbb {P} (\widetilde {O} _ {t} \in \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau})) \\ = d _ {T V} \left(\mathbb {P} \left(\left(s _ {t}, a _ {t}\right) \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mathbb {P} \left(\left(\widetilde {s} _ {t}, \widetilde {a} _ {t}\right) \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right)\right) \\ \leq d _ {T V} (\mathbb {P} (s _ {t} \in \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}), \mathbb {P} (\widetilde {s} _ {t} \in \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau})) + \frac {1}{2} L _ {\pi} \mathbb {E} \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \| \\ \leq d _ {T V} \left(\mathbb {P} \left(O _ {t - 1} \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mathbb {P} \left(\widetilde {O} _ {t - 1} \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right)\right) + \frac {1}{2} L _ {\pi} \mathbb {E} \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \|. \\ \end{array}
$$
Repeat the above argument from  $t$  to  $t - \tau + 1$ , we have
$$
d _ {T V} \left(\mathbb {P} \left(O _ {t} \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mathbb {P} \left(\widetilde {O} _ {t} \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right)\right) \leq \frac {1}{2} | \mathcal {A} | L _ {\pi} \sum_ {k = t - \tau} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} _ {t - \tau} \|. \tag {33}
$$
Plugging (33) into (32), we have
$$
| \mathbb {E} [ \Phi (O _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) - \mathbb {E} [ \Phi (\widetilde {O} _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] | \leq 2 U _ {r} ^ {2} | \mathcal {A} | L _ {\pi} \sum_ {k = t - \tau} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} _ {t - \tau} \|.
$$
Step 4: show that conditioned on  $s_{t - \tau +1}$  and  $\theta_{t - \tau}$ , we have
$$
\mathbb {E} \left[ \Phi \left(\widetilde {O} _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \leq 4 U _ {r} ^ {2} m \rho^ {\tau - 1}. \tag {34}
$$
Note that according to definition, we have
$$
\mathbb {E} [ \Phi (O _ {t - \tau} ^ {\prime}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) | \boldsymbol {\theta} _ {t - \tau} ] = 0,
$$
where  $O_{t - \tau}^{\prime} = (s_{t - \tau}^{\prime}, a_{t - \tau}^{\prime}, s_{t - \tau + 1}^{\prime})$  is the tuple generated by  $s_{t - \tau}^{\prime} \sim \mu_{t - \tau}, a_{t - \tau}^{\prime} \sim \pi_{\theta_{t - \tau}}, s_{t - \tau + 1}^{\prime} \sim \mathcal{P}$ . From the uniform ergodicity in Assumption 3.2, it shows that
$$
d _ {T V} \left(\mathbb {P} \left(\widetilde {s} _ {t} = \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mu_ {\boldsymbol {\theta} _ {t - \tau}}\right) \leq m \rho^ {\tau - 1}.
$$
$$
\begin{array}{l} \mathbb {E} \left[ \Phi \left(\widetilde {O} _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] = \mathbb {E} \left[ \Phi \left(\widetilde {O} _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) - \Phi \left(O _ {t - \tau} ^ {\prime}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \\ = \mathbb {E} [ (\eta_ {t - \tau} - J (\boldsymbol {\theta} _ {t - \tau})) (r (\tilde {s} _ {t}, \tilde {a} _ {t}) - r (s _ {t - \tau} ^ {\prime}, a _ {t - \tau} ^ {\prime})) ] \\ \leq 4 U _ {r} ^ {2} d _ {T V} \left(\mathbb {P} \left(\widetilde {O} _ {t - \tau} = \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mu_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \pi_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \mathcal {P}\right) \\ \leq 4 U _ {r} ^ {2} m \rho^ {\tau - 1}. \\ \end{array}
$$
Combing (29), (30), (31), and (34), we have
$$
\begin{array}{l} \mathbb {E} \left[ \Phi \left(O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t}\right) \right] = \mathbb {E} \left[ \Phi \left(O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t}\right) - \Phi \left(O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t - \tau}\right) \right] + \mathbb {E} \left[ \Phi \left(O _ {t}, \eta_ {t}, \boldsymbol {\theta} _ {t - \tau}\right) - \Phi \left(O _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \\ + \mathbb {E} [ \Phi (O _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) - \Phi (\widetilde {O} _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] + \mathbb {E} [ \Phi (\widetilde {O} _ {t}, \eta_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] \\ \leq 4 U _ {r} L _ {J} \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \right\| + 2 U _ {r} \left| \eta_ {t} - \eta_ {t - \tau} \right| + 2 U _ {r} ^ {2} | \mathcal {A} | L _ {\pi} \sum_ {i = t - \tau} ^ {t} \mathbb {E} \left\| \boldsymbol {\theta} _ {i} - \boldsymbol {\theta} _ {t - \tau} \right\| \\ + 4 U _ {r} ^ {2} m \rho^ {\tau - 1}, \\ \end{array}
$$
Proof of Lemma C.3.
Step 1: show that for any  $\theta_{1},\theta_{2},\omega$  and tuple  $O = (s,a,s')$ , we have
$$
\left| \Psi (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) - \Psi (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \leq C _ {1} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|, \right. \tag {35}
$$
By definition of  $\Psi (O,\omega ,\theta)$  in (13), we have