SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} | \Psi (O, \omega , \theta_ {1}) - \Psi (O, \omega , \theta_ {2}) | \\ = | \langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*}, g (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) - \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) \rangle - \langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {2} ^ {*}, g (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) - \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \rangle | \\ \leq \underbrace {\left| \left\langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*} , g (O , \boldsymbol {\omega} , \boldsymbol {\theta} _ {1}) - \bar {g} (\boldsymbol {\omega} , \boldsymbol {\theta} _ {1}) \right\rangle - \left\langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*} , g (O , \boldsymbol {\omega} , \boldsymbol {\theta} _ {2}) - \bar {g} (\boldsymbol {\omega} , \boldsymbol {\theta} _ {2}) \right\rangle \right|} _ {I _ {1}} \\ + \underbrace {\left| \left\langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*} , g (O , \boldsymbol {\omega} , \boldsymbol {\theta} _ {2}) - \bar {g} (\boldsymbol {\omega} , \boldsymbol {\theta} _ {2}) \right\rangle - \left\langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {2} ^ {*} , g (O , \boldsymbol {\omega} , \boldsymbol {\theta} _ {2}) - \bar {g} (\boldsymbol {\omega} , \boldsymbol {\theta} _ {2}) \right\rangle \right|}. _ {I _ {2}} \\ \end{array}
$$
For term  $I_{1}$  , we have
$$
\begin{array}{l} I _ {1} = \left| \left\langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*}, g (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) - \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) \right\rangle - \left\langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*}, g (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) - \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \right\rangle \right| \\ = | \langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*}, g (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) - g (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \rangle | + | \langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*}, \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) - \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \rangle | \\ = | \langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*}, \phi (s) (J (\boldsymbol {\theta} _ {1}) - J (\boldsymbol {\theta} _ {2})) \rangle | + | \langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*}, \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) - \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \rangle | \\ \leq 2 U _ {\omega} L _ {J} \left\| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \right\| + 2 U _ {\omega} \left\| \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) - \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \right\| \\ \leq 2 U _ {\omega} L _ {J} \left\| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \right\| + 2 U _ {\omega} \cdot 2 U _ {\delta} d _ {T V} \left(\mu_ {\boldsymbol {\theta} _ {1}} \otimes \pi_ {\boldsymbol {\theta} _ {1}} \otimes \mathcal {P}, \mu_ {\boldsymbol {\theta} _ {2}} \otimes \pi_ {\boldsymbol {\theta} _ {2}} \otimes \mathcal {P}\right) \\ \leq 2 U _ {\omega} L _ {J} \left\| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \right\| + 2 U _ {\delta} ^ {2} d _ {T V} \left(\mu_ {\boldsymbol {\theta} _ {1}} \otimes \pi_ {\boldsymbol {\theta} _ {1}} \otimes \mathcal {P}, \mu_ {\boldsymbol {\theta} _ {2}} \otimes \pi_ {\boldsymbol {\theta} _ {2}} \otimes \mathcal {P}\right) \\ \leq \left( \right.2 U _ {\delta} L _ {J} + 2 U _ {\delta} ^ {2} | \mathcal {A} | L _ {\pi} \left(1 + \lceil \log_ {\rho} m ^ {- 1} \rceil + \frac {1}{1 - \rho}\right) \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \rVert , \\ \end{array}
$$
where we use the fact that  $U_{\delta} = 2U_{r} + 2U_{\omega}$  and the last inequality comes from Lemma B.5.
For term  $I_{2}$ , from Cauchy-Schwartz inequality, we have
$$
\begin{array}{l} I _ {2} = \left| \left\langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {1} ^ {*}, g (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) - \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \right\rangle - \left\langle \boldsymbol {\omega} - \boldsymbol {\omega} _ {2} ^ {*}, g (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) - \bar {g} (\boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \right\rangle \right| \\ = | \langle \omega_ {1} ^ {*} - \omega_ {2} ^ {*}, g (O, \omega , \theta_ {2}) - \bar {g} (\omega , \theta_ {2}) \rangle | \\ \leq 2 U _ {\delta} \left\| \omega_ {1} ^ {*} - \omega_ {2} ^ {*} \right\| \\ \leq 2 U _ {\delta} L _ {*} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|. \\ \end{array}
$$
Combining the results from  $I_{1}$  and  $I_{2}$ , we get
$$
| \Psi (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {1}) - \Psi (O, \boldsymbol {\omega}, \boldsymbol {\theta} _ {2}) \leq C _ {1} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|,
$$
where  $C_1 = 2U_\delta^2 |\mathcal{A}|L_\pi \big(1 + \lceil \log_\rho m^{-1}\rceil +\frac{1}{1 - \rho}\big) + 2U_\delta L_J + 2U_\delta L_*$
Step 2: show that for any  $\theta, \omega_1, \omega_2$  and tuple  $O(s, a, s')$ , we have
$$
\left| \Psi (O, \omega_ {1}, \boldsymbol {\theta}) - \Psi (O, \omega_ {2}, \boldsymbol {\theta}) \right| \leq 6 U _ {\delta} \| \omega_ {1} - \omega_ {2} \|. \tag {36}
$$
$$
\begin{array}{l} | \Psi (O, \omega_ {1}, \boldsymbol {\theta}) - \Psi (O, \omega_ {2}, \boldsymbol {\theta}) | \\ = | \langle \omega_ {1} - \omega^ {*}, g (O, \omega_ {1}, \boldsymbol {\theta}) - \bar {g} (\omega_ {1}, \boldsymbol {\theta}) \rangle - \langle \omega_ {2} - \omega^ {*}, g (O, \omega_ {2}, \boldsymbol {\theta}) - \bar {g} (\omega_ {2}, \boldsymbol {\theta}) \rangle | \\ \leq | \langle \omega_ {1} - \omega^ {*}, g (O, \omega_ {1}, \boldsymbol {\theta}) - \bar {g} (\omega_ {1}, \boldsymbol {\theta}) \rangle - \langle \omega_ {1} - \omega^ {*}, g (O, \omega_ {2}, \boldsymbol {\theta}) - \bar {g} (\omega_ {2}, \boldsymbol {\theta}) \rangle | \\ + \left| \left\langle \omega_ {1} - \omega^ {*}, g (O, \omega_ {2}, \theta) - \bar {g} (\omega_ {2}, \theta) \right\rangle - \left\langle \omega_ {2} - \omega^ {*}, g (O, \omega_ {2}, \theta) - \bar {g} (\omega_ {2}, \theta) \right\rangle \right| \\ \leq 2 U _ {\omega} \left\| \left(g (O, \omega_ {1}) - g (O, \omega_ {2})\right) - (\bar {g} (\omega_ {1}, \theta) - \bar {g} (\omega_ {2}, \theta)) \right\| + 2 U _ {\delta} \left\| \omega_ {1} - \omega_ {2} \right\| \\ \leq 6 U _ {\delta} \| \omega_ {1} - \omega_ {2} \|, \\ \end{array}
$$
where the last inequality is due to  $\| g(O,\omega_1,\pmb {\theta}) - g(O,\omega_2,\pmb {\theta})\| = |(\phi (s') - \phi (s))^{\top}(\omega_1 - \omega_2)|\leq 2\| \omega_1 - \omega_2\|$ $\| \bar{g} (\omega_1,\pmb {\theta}) - \bar{g} (\omega_2,\pmb {\theta})\| \leq 2\| \omega_1 - \omega_2\|$  , and  $2U_{\omega}\leq U_{\delta}$
Step 3: show that for tuples  $O_{t} = (s_{t},a_{t},s_{t + 1})$  and  $\widetilde{O}_t = (\widetilde{s}_t,\widetilde{a}_t,\widetilde{s}_{t + 1})$ . Conditioning on  $s_{t - \tau +1}$  and  $\theta_{t - \tau}$ , we have
$$
\mathbb {E} \left[ \Psi \left(O _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) - \Psi \left(\widetilde {O} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \leq U _ {\delta} ^ {2} | \mathcal {A} | L _ {\pi} \sum_ {k = t - \tau} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} _ {t - \tau} \|. \tag {37}
$$
By the definition of total variation norm, we have
$$
\begin{array}{l} \mathbb {E} \left[ \Psi \left(O _ {t}, \omega_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) - \Psi \left(\widetilde {O} _ {t}, \omega_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \\ \leq \mathbb {E} [ \left\langle \boldsymbol {\omega} _ {t - \tau} - \boldsymbol {\omega} _ {t - \tau} ^ {*}, g \left(O _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) - g \left(\bar {O} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right\rangle ] \\ \leq 2 U _ {\delta} ^ {2} d _ {T V} \left(\mathbb {P} \left(O _ {t} \in \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {- \tau}\right), \mathbb {P} \left(\tilde {O} _ {t} \in \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right)\right) \\ \stackrel {(1)} {\leq} U _ {\delta} ^ {2} | \mathcal {A} | L _ {\pi} \sum_ {k = t - \tau} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} _ {t - \tau} \| \\ \leq U _ {\delta} ^ {2} | \mathcal {A} | L _ {\pi} G \tau (\tau + 1) \alpha_ {t - \tau}, \\ \end{array}
$$
where (1) follows from (33).
Step 4: show that conditioning on  $s_{t - \tau +1}$  and  $\theta_{t - \tau}$
$$
\mathbb {E} \left[ \Psi \left(\widetilde {O} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \leq 2 U _ {\delta} ^ {2} m \rho^ {\tau - 1} \tag {38}
$$
From the definition of  $\Psi (O,\omega ,\theta)$  , we have
$$
\mathbb {E} \left[ \Psi \left(O _ {t - \tau} ^ {\prime}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau} \right] = 0,
$$
where  $O_{t - \tau}^{\prime}$  is the tuple generated by  $s_{t - \tau}^{\prime}\sim \mu_{\pmb{\theta}_{t - \tau}},a_{t - \tau}^{\prime}\sim \pi_{\pmb{\theta}_{t - \tau}},s_{t - \tau +1}^{\prime}\sim \mathcal{P}$ . From Assumption 3.2, we have
Then, it holds that
$$
\begin{array}{l} \mathbb {E} [ \Psi (\widetilde {\boldsymbol {O}} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] = \mathbb {E} [ \Psi (\widetilde {\boldsymbol {O}} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) - \Psi (O _ {t - \tau} ^ {\prime}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] \\ = \mathbb {E} \left\langle \omega_ {t - \tau} - \omega_ {t - \tau} ^ {*}, g \left(\widetilde {O} _ {t}, \omega_ {t - \tau}, \theta_ {t - \tau} - g \left(O _ {t - \tau} ^ {\prime}, \omega_ {t - \tau}, \theta_ {t - \tau}\right) \right. \right\rangle \\ \leq 4 U _ {\omega} U _ {\delta} d _ {T V} \left(\mathbb {P} \left(\widetilde {O} _ {t} = \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mu_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \pi_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \mathcal {P}\right) \\ \leq 2 U _ {\delta} ^ {2} d _ {T V} \left(\mathbb {P} \left(\widetilde {O} _ {t} = \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mu_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \pi_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \mathcal {P}\right) \\ = 2 U _ {\delta} ^ {2} d _ {T V} \left(\mathbb {P} \left(\left(\widetilde {s} _ {t}, \widetilde {a} _ {t}\right) \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mu_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \pi_ {\boldsymbol {\theta} _ {t - \tau}}\right) \\ = 2 U _ {\delta} ^ {2} d _ {T V} \left(\mathbb {P} \left(\widetilde {s} _ {t} = \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mu_ {\boldsymbol {\theta} _ {t - \tau}}\right) \\ \leq 2 U _ {\delta} ^ {2} m \rho^ {\tau - 1}. \\ \end{array}
$$
Combining (35), (36), (37), and (38), we have
$$
\begin{array}{l} \mathbb {E} \left[ \Psi \left(O _ {t}, \omega_ {t}, \boldsymbol {\theta} _ {t}\right) \right] = \mathbb {E} \left[ \Psi \left(O _ {t}, \omega_ {t}, \boldsymbol {\theta} _ {t}\right) - \Psi \left(O _ {t}, \omega_ {t}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \\ + \mathbb {E} [ \Psi (O _ {t}, \omega_ {t}, \boldsymbol {\theta} _ {t - \tau}) - \Psi (O _ {t}, \omega_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] \\ + \mathbb {E} [ \Psi (O _ {t}, \omega_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) - \Psi (\widetilde {O} _ {t}, \omega_ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] \\ + \mathbb {E} [ \Psi (\widetilde {\mathcal {O}} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] \\ \leq C _ {1} \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \right\| + U _ {\delta} ^ {2} | \mathcal {A} | L _ {\pi} G \tau (\tau + 1) \alpha_ {t - \tau} \\ + 2 U _ {\delta} ^ {2} m \rho^ {\tau - 1} + 6 U _ {\delta} \| \boldsymbol {\omega} _ {t} - \boldsymbol {\omega} _ {t - \tau} \|, \\ \end{array}
$$
where  $C_1 = 2U_\delta^2 |\mathcal{A}|L_\pi (1 + \lceil \log_\rho m^{-1}\rceil +\frac{1}{1 - \rho}) + 2U_\delta (L_J + L_*)$
Proof of Lemma C.4.
Step 1: show that
$$
\left\| \Xi \left(O _ {t}, \boldsymbol {\theta} _ {t}, \boldsymbol {\theta} _ {t - \tau}\right) - \Xi \left(O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t - \tau}\right) \right\| \leq \left(3 U _ {\delta} L _ {h} + 2 U _ {\delta} B L _ {*}\right) \| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \| \tag {39}
$$
Since  $\Xi(O_t, \omega_t, \theta_t) = \langle \omega_t - \omega_t^*, \mathbb{E}_{O'}[h(O', \theta)] - h(O, \theta) \rangle$ , we define  $\mathbb{E}_{\pmb{\theta}}[h(O', \pmb{\theta})] := \mathbb{E}_{O'}[h(O', \pmb{\theta})]$ , where  $\mathbb{E}_{\pmb{\theta}}$  is the shorthand of  $\mathbb{E}_{O' \sim (\mu_{\theta}, \pi_{\theta}, \mathcal{P})}$ . In the following, we will show that each term in  $\Xi(O_t, \omega_t, \theta_t)$  is Lipschitz.
Term  $\omega_{t}$  is not related to  $\pmb{\theta}$  and term  $\omega_{t}^{*} := \omega^{*}(\pmb{\theta}_{t})$  is  $L_{*}$ -Lipschitz.
For term  $h(O, \pmb{\theta})$ , denote  $\delta(O, \pmb{\theta}) \coloneqq r(s, a) - r(\pmb{\theta}) + (\phi(s') - \phi(s))^\top \pmb{\omega}^*$ , we have
$$
\begin{array}{l} \left\| h \left(O, \boldsymbol {\theta} _ {1}\right) - h \left(O, \boldsymbol {\theta} _ {2}\right) \right\| \\ = \| \delta (O, \boldsymbol {\theta} _ {1}) \nabla \log \pi_ {\boldsymbol {\theta} _ {1}} (a | s) - \delta (O _ {t}, \boldsymbol {\theta} _ {2}) \nabla \log \pi_ {\boldsymbol {\theta} _ {2}} (a | s) \| \\ \leq \left\| \delta (O, \boldsymbol {\theta} _ {1}) \nabla \log \pi_ {\boldsymbol {\theta} _ {1}} (a | s) - \delta (O, \boldsymbol {\theta} _ {1}) \nabla \log \pi_ {\boldsymbol {\theta} _ {2}} (a | s) \right\| \\ + \left\| \delta (O, \boldsymbol {\theta} _ {1}) \nabla \log \pi_ {\boldsymbol {\theta} _ {2}} (a | s) - \delta (O, \boldsymbol {\theta} _ {2}) \nabla \log \pi_ {\boldsymbol {\theta} _ {2}} (a | s) \right\| \\ \leq U _ {\delta} L _ {l} \left\| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \right\| + B | \delta (O, \boldsymbol {\theta} _ {1}) - \delta (O, \boldsymbol {\theta} _ {2}) | \\ \leq U _ {\delta} L _ {l} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \| + B (| r (\boldsymbol {\theta} _ {1}) - r (\boldsymbol {\theta} _ {2}) | + \| \phi (s ^ {\prime}) - \phi (s) \| \cdot \| \boldsymbol {\omega} ^ {*} (\boldsymbol {\theta} _ {1}) - \boldsymbol {\omega} ^ {*} (\boldsymbol {\theta} _ {2}) \|) \\ \leq \left(U _ {\delta} L _ {l} + 2 B L _ {*}\right) \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \| + B | \mathbb {E} _ {s \sim \mu_ {\boldsymbol {\theta} _ {1}}, a \sim \pi_ {\boldsymbol {\theta} _ {1}}} [ r (s, a) ] - \mathbb {E} _ {s \sim \mu_ {\boldsymbol {\theta} _ {1}}, a \sim \pi_ {\boldsymbol {\theta} _ {2}}} [ r (s, a) ] | \\ \leq \left(U _ {\delta} L _ {l} + 2 B L _ {*}\right) \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \| + 2 B U _ {r} d _ {T V} \left(\mu_ {\boldsymbol {\theta} _ {1}} \otimes \pi_ {\boldsymbol {\theta} _ {1}}, \mu_ {\boldsymbol {\theta} _ {2}} \otimes \pi_ {\boldsymbol {\theta} _ {2}}\right) \\ \leq \left(U _ {\delta} L _ {l} + 2 B L _ {*} + 2 B U _ {r} \mid \mathcal {A} \mid L _ {\pi} \left(1 + \lceil \log_ {\rho} m ^ {- 1} \rceil + \frac {1}{1 - \rho}\right)\right) \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|. \\ \end{array}
$$
Hence we have  $h(O, \theta)$  is  $L_h$ -Lipschitz, where  $L_h$  denotes the above coefficient.
For term  $\mathbb{E}_{\pmb{\theta}}[h(O',\pmb{\theta})]$ , we have
$$
\begin{array}{l} \left\| \mathbb {E} _ {\boldsymbol {\theta} _ {1}} \left[ h \left(O ^ {\prime}, \boldsymbol {\theta} _ {1}\right) \right] - \mathbb {E} _ {\boldsymbol {\theta} _ {2}} \left[ h \left(O ^ {\prime}, \boldsymbol {\theta} _ {2}\right) \right] \right\| \\ \leq \left\| \mathbb {E} _ {\boldsymbol {\theta} _ {1}} \left[ h \left(O ^ {\prime}, \boldsymbol {\theta} _ {1}\right) \right] - \mathbb {E} _ {\boldsymbol {\theta} _ {1}} \left[ h \left(O ^ {\prime}, \boldsymbol {\theta} _ {2}\right) \right] \right\| + \left\| \mathbb {E} _ {\boldsymbol {\theta} _ {1}} \left[ h \left(O ^ {\prime}, \boldsymbol {\theta} _ {2}\right) \right] - \mathbb {E} _ {\boldsymbol {\theta} _ {2}} \left[ h \left(O ^ {\prime}, \boldsymbol {\theta} _ {2}\right) \right] \right\| \\ \leq \mathbb {E} _ {\boldsymbol {\theta} _ {1}} [ \| h (O ^ {\prime}, \boldsymbol {\theta} _ {1}) - h (O ^ {\prime}, \boldsymbol {\theta} _ {2}) \| ] + \| \mathbb {E} _ {\boldsymbol {\theta} _ {1}} [ h (O ^ {\prime}, \boldsymbol {\theta} _ {2}) ] - \mathbb {E} _ {\boldsymbol {\theta} _ {2}} [ h (O ^ {\prime}, \boldsymbol {\theta} _ {2}) ] \| \\ \leq L _ {h} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \| + \| \mathbb {E} _ {\boldsymbol {\theta} _ {1}} [ h (O ^ {\prime}, \boldsymbol {\theta} _ {2}) ] - \mathbb {E} _ {\boldsymbol {\theta} _ {2}} [ h (O ^ {\prime}, \boldsymbol {\theta} _ {2}) ] \| \\ \leq L _ {h} \left\| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \right\| + 2 B U _ {r} d _ {T V} \left(\mu_ {\boldsymbol {\theta} _ {1}} \otimes \pi_ {\boldsymbol {\theta} _ {1}}, \mu_ {\boldsymbol {\theta} _ {2}} \otimes \pi_ {\boldsymbol {\theta} _ {2}}\right) \\ \leq \left[ L _ {h} + 2 B U _ {r} | \mathcal {A} | L _ {\pi} \left(1 + \lceil \log_ {\rho} m ^ {- 1} \rceil + \frac {1}{1 - \rho}\right) \right] \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \| \\ \leq 2 L _ {h} \| \boldsymbol {\theta} _ {1} - \boldsymbol {\theta} _ {2} \|. \\ \end{array}
$$
Then we have  $\omega_{t} - \omega_{t}^{*}$  is  $U_{\delta}$ -bounded and  $L_{*}$ -Lipschitz;  $h(O, \theta) - \mathbb{E}_{\theta}[h(O', \theta)]$  is  $3L_{h}$ -Lipschitz and  $2U_{\delta}B$ -bounded. By the triangle inequality, we have
$$
\left\| \Xi \left(O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t}\right) - \Xi \left(O _ {t}, \boldsymbol {\omega} _ {t}, \boldsymbol {\theta} _ {t - \tau}\right) \right\| \leq \left(3 U _ {\delta} L _ {h} + 2 U _ {\delta} B L _ {*}\right) \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \right\| \leq C _ {2} \left\| \boldsymbol {\theta} _ {t} - \boldsymbol {\theta} _ {t - \tau} \right\|,
$$
Step 2: show that
$$
\left\| \Xi \left(O, \omega_ {t}, \boldsymbol {\theta}\right) - \Xi \left(O, \omega_ {t - \tau}, \boldsymbol {\theta}\right) \right\| \leq 2 U _ {\delta} B \| \omega_ {t} - \omega_ {t - \tau} \|. \tag {40}
$$
Actually, we have
$$
\left\| \Xi (O, \omega_ {1}, \boldsymbol {\theta}) - \Xi (O, \omega_ {2}, \boldsymbol {\theta}) \right\| = \| \langle \omega_ {1} - \omega_ {2}, \mathbb {E} _ {O ^ {\prime}} [ h (O ^ {\prime}, \boldsymbol {\theta}) ] - h (O, \boldsymbol {\theta}) \rangle \| \leq 2 U _ {\delta} B \| \omega_ {1} - \omega_ {2} \|
$$
$$
\mathbb {E} \left[ \Xi \left(O _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) - \Xi \left(\widetilde {O} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \leq 2 U _ {\delta} ^ {2} B | \mathcal {A} | L _ {\pi} \sum_ {k = t - \tau} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} _ {t - \tau} \|. \tag {41}
$$
By definition of  $\Xi (O,\omega ,\theta)$  , we have
$$
\begin{array}{l} \left\| \mathbb {E} \left[ \Xi \left(O _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) - \Xi \left(\widetilde {O} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \right\| \\ = \| \mathbb {E} [ \left\langle \boldsymbol {\omega} _ {t - \tau} - \boldsymbol {\omega} _ {t - \tau} ^ {*}, h (\tilde {O} _ {t}, \boldsymbol {\theta} _ {t - \tau}) - h (O _ {t}, \boldsymbol {\theta} _ {t - \tau}) \right\rangle \| \\ = \| \mathbb {E} [ \langle \boldsymbol {\omega} _ {t - \tau} - \boldsymbol {\omega} _ {t - \tau} ^ {*}, h (\widetilde {O} _ {t}, \boldsymbol {\theta} _ {t - \tau}) \rangle - \mathbb {E} [ \langle \boldsymbol {\omega} _ {t - \tau} - \boldsymbol {\omega} _ {t - \tau} ^ {*}, h (O _ {t}, \boldsymbol {\theta} _ {t - \tau}) \rangle ] \| \\ \leq 4 U _ {\delta} ^ {2} B d _ {T V} \left(\mathbb {P} \left(O _ {t} \in \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right), \mathbb {P} \left(\widetilde {O} _ {t} \in \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right)\right), \tag {42} \\ \end{array}
$$
where the inequality comes from the definition of total variation distance. The total variation norm between  $O_{t}$  and  $\widetilde{O}_{t}$  has been computed in (33). Plugging (33) into (42), we get
$$
\begin{array}{l} \| \mathbb {E} [ \Xi (O _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) - \Xi (\widetilde {O} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] \| \leq 2 U _ {\delta} ^ {2} B | \mathcal {A} | L _ {\pi} \sum_ {k = t - \tau} ^ {t} \mathbb {E} \| \boldsymbol {\theta} _ {k} - \boldsymbol {\theta} _ {t - \tau} \| \\ \leq 2 U _ {\delta} ^ {2} B | \mathcal {A} | L _ {\pi} G \tau (\tau + 1) \alpha_ {t - \tau}. \\ \end{array}
$$
Step 4: Show that conditioning on  $s_{t - \tau +1}$  and  $\theta_{t - \tau}$ , we have
$$
\left\| \mathbb {E} \left[ \Xi \left(\widetilde {O} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}\right) \right] \right\| \leq 4 U _ {\delta} ^ {2} B m \rho^ {\tau - 1}. \tag {43}
$$
It can be shown that
$$
\begin{array}{l} \| \mathbb {E} [ \Xi (\widetilde {\mathcal {O}} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] \| \stackrel {(1)} {=} \| \mathbb {E} [ \Xi (\widetilde {\mathcal {O}} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) - \Xi (O _ {t - \tau} ^ {\prime}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) ] \| \\ = \| \mathbb {E} [ \left\langle \boldsymbol {\omega} _ {t - \tau} - \boldsymbol {\omega} _ {t - \tau} ^ {*}, h \left(O _ {t - \tau} ^ {\prime}, \boldsymbol {\theta} _ {t - \tau}\right) \right\rangle - \left\langle \boldsymbol {\omega} _ {t - \tau} - \boldsymbol {\omega} _ {t - \tau} ^ {*}, h \left(\widetilde {O} _ {t}, \boldsymbol {\theta} _ {t - \tau}\right) \right\rangle ] \| \\ \end{array}
$$
$$
\stackrel {(2)} {\leq} 4 U _ {\delta} ^ {2} B d _ {T V} (\mathbb {P} (\widetilde {O} _ {t} \in \cdot | s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}), \mu_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \pi_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \mathcal {P}),
$$
where (1) is due to the fact that  $O_{t}^{\prime}$  is from the stationary distribution which satisfies  $\mathbb{E}[\Xi(O_{t}', \boldsymbol{\omega}_{t - \tau}, \boldsymbol{\theta}_{t - \tau})] = 0$  and (2) follows from the definition of total variation distance. From Assumption 3.2, we know that
$$
d _ {T V} \left(\mathbb {P} \left(\widetilde {s} _ {t} \in \cdot\right), \mu_ {\boldsymbol {\theta} _ {t - \tau}}\right) \leq m \rho^ {\tau - 1}.
$$
We also have the fact that
$$
\mathbb {P} \left(\widetilde {O} _ {t} \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right) = \mathbb {P} \left(\widetilde {s} _ {t} \in \cdot \mid s _ {t - \tau + 1}, \boldsymbol {\theta} _ {t - \tau}\right) \otimes \pi_ {\boldsymbol {\theta} _ {t - \tau}} \otimes \mathcal {P}.
$$
$$
\| \mathbb {E} [ \Xi (\tilde {O} _ {t}, \boldsymbol {\omega} _ {t - \tau}, \boldsymbol {\theta} _ {t - \tau}) \| \leq 4 U _ {\delta} ^ {2} B m \rho^ {\tau - 1}.
$$
Combining (39)-(43), we can decompose the Markovian bias as