SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\left| \left| P \left(O _ {t} \in \cdot\right) - P \left(\tilde {O} _ {t} \in \cdot\right) \right| \right| _ {1} \leq 2 n _ {a} L _ {\pi} \sum_ {i = t - \tau_ {\mathrm {m i x}}} ^ {t} \mathbb {E} \left[ \left| \left| \theta_ {i} - \theta_ {t - \tau_ {\mathrm {m i x}}} \right| \right| \right].
$$
Proof. For the first part,
$$
\begin{array}{l} \left\| P \left(S _ {\tau + 1} \in \cdot\right) - P \left(\tilde {S} _ {\tau + 1} \in \cdot\right) \right\| _ {1} \\ = \sum_ {s ^ {\prime}} \left| P \left(S _ {\tau + 1} = s ^ {\prime}\right) - P \left(\tilde {S} _ {\tau + 1} = s ^ {\prime}\right) \right| \\ \leq \sum_ {s, a, s ^ {\prime}, a ^ {\prime}} \left| P \left(S _ {\tau} = s, A _ {\tau} = a, S _ {\tau + 1} = s ^ {\prime} A _ {\tau + 1} = a ^ {\prime}\right) - P \left(\tilde {S} _ {\tau} = s, \tilde {A} _ {\tau} = a, \tilde {S} _ {\tau + 1} = s ^ {\prime}, \tilde {A} _ {\tau + 1} = a ^ {\prime}\right) \right| \\ = \left\| P \left(O _ {\tau} \in \cdot\right) - P \left(\tilde {O} _ {\tau} \in \cdot\right) \right\| _ {1}. \\ \end{array}
$$
For the second part, conditioned on  $\theta_{t - \tau_{\mathrm{mix}}}$  and  $s_t - \tau_{\mathrm{mix}} + 1$  , we denote
$$
M _ {1} = P \left(S _ {\tau} = s\right) P \left(\theta_ {\tau} = z _ {\tau} \mid S _ {\tau} = s\right) \pi (a \mid s, z _ {\tau}) P _ {\text {e n v}} \left(s ^ {\prime} \mid s, a\right) \pi \left(a ^ {\prime} \mid s ^ {\prime}, z _ {\tau}\right)
$$
$$
M _ {2} = P \left(S _ {\tau} = s\right) P \left(\theta_ {\tau} = z _ {\tau} \mid S _ {\tau} = s\right) \pi (a \mid s, z _ {\tau}) P _ {\text {e n v}} \left(s ^ {\prime} \mid s, a\right) \pi \left(a ^ {\prime} \mid s ^ {\prime}, \theta_ {t - \tau_ {\text {m i x}}}\right)
$$
$$
M _ {3} = P \left(S _ {\tau} = s\right) P \left(\theta_ {\tau} = z _ {\tau} \mid S _ {\tau} = s\right) \pi \left(a \mid s, \theta_ {t - \tau_ {\operatorname {m i x}}}\right) P _ {\text {e n v}} \left(s ^ {\prime} \mid s, a\right) \pi \left(a ^ {\prime} \mid s ^ {\prime}, \theta_ {t - \tau_ {\operatorname {m i x}}}\right)
$$
$$
M _ {4} = P (\tilde {S} _ {\tau} = s) P \left(\theta_ {\tau} = z _ {\tau} \mid S _ {\tau} = s\right) \pi \left(a \mid s, \theta_ {t - \tau_ {\operatorname {m i x}}}\right) P _ {\text {e n v}} \left(s ^ {\prime} \mid s, a\right) \pi \left(a ^ {\prime} \mid s ^ {\prime}, \theta_ {t - \tau_ {\operatorname {m i x}}}\right),
$$
which will be useful later. Using notations from the original Markov Chain,
$$
\begin{array}{l} P \left(S _ {\tau} = s, A _ {\tau} = a, S _ {\tau + 1} = s ^ {\prime}, A _ {\tau + 1} = a ^ {\prime}\right) \\ = \int P (S _ {\tau} = s, A _ {\tau} = a, S _ {\tau + 1} = s ^ {\prime}, A _ {\tau + 1} = a ^ {\prime}, \theta_ {\tau} = z _ {\tau}) d z _ {\tau} \\ = \int P (S _ {\tau} = s) P (\theta_ {\tau} = z _ {\tau} | S _ {\tau} = s) \pi (a | s, z _ {\tau}) P _ {\mathrm {e n v}} (s ^ {\prime} | s, a) \pi (a ^ {\prime} | s ^ {\prime}, z _ {\tau}) d z _ {\tau} \\ = \int M _ {1} d z _ {\tau}. \\ \end{array}
$$
Similarly, the auxiliary Markov Chain gives us
$$
\begin{array}{l} P \left(\tilde {S} _ {\tau} = s, \tilde {A} _ {\tau} = a, \tilde {S} _ {\tau + 1} = s ^ {\prime}, \tilde {A} _ {\tau + 1} = a ^ {\prime}\right) \\ = P \left(\tilde {S} _ {\tau} = s\right) \pi (a | s, \theta_ {t - \tau_ {\operatorname {m i x}}}) P _ {\text {e n v}} \left(s ^ {\prime} \mid s, a\right) \pi \left(a ^ {\prime} \mid s ^ {\prime}, \theta_ {t - \tau_ {\operatorname {m i x}}}\right) \\ = \int P (\tilde {S} _ {\tau} = s) P (\theta_ {\tau} = z _ {\tau} | S _ {\tau} = s) \pi (a | s, \theta_ {t - \tau_ {\mathrm {m i x}}}) P _ {\mathrm {e n v}} (s ^ {\prime} | s, a) \pi (a ^ {\prime} | s ^ {\prime}, \theta_ {t - \tau_ {\mathrm {m i x}}}) d z _ {\tau} \\ = \int M _ {4} d z _ {\tau}. \\ \end{array}
$$
We now rephrase the left-hand side term we want to prove,
$$
\begin{array}{l} \left| \left| P \left(O _ {\tau} \in \cdot\right) - P \left(\tilde {O} _ {\tau} \in \cdot\right) \right| \right| _ {1} \\ \leq \sum_ {s, a, s ^ {\prime}, a ^ {\prime}} \left| P \left(S _ {\tau} = s, A _ {\tau} = a, S _ {\tau + 1} = s ^ {\prime}, A _ {\tau + 1} = a ^ {\prime}\right) - P \left(\tilde {S} _ {\tau} = s, \tilde {A} _ {\tau} = a, \tilde {S} _ {\tau + 1} = s ^ {\prime}, \tilde {A} _ {\tau + 1} = a ^ {\prime}\right) \right| \\ \leq \sum_ {s, a, s ^ {\prime}, a ^ {\prime}} \int | M _ {1} - M _ {4} | d z _ {\tau} \\ \leq \underbrace{\sum_{s,a,s^{\prime},a^{\prime}}\int|M_{1} - M_{2}|dz_{\tau}}_{I_{1}} + \underbrace{\sum_{s,a,s^{\prime},a^{\prime}}\int|M_{2} - M_{3}|dz_{\tau}}_{I_{2}} + \underbrace{\sum_{s,a,s^{\prime},a^{\prime}}\int|M_{3} - M_{4}|dz_{\tau}}_{I_{3}}. \\ \end{array}
$$
For  $I_{1}$
$$
\begin{array}{l} I _ {1} = \sum_ {s, a, s ^ {\prime}, a ^ {\prime}} \int P (S _ {\tau} = s) P (\theta_ {\tau} = z _ {\tau} | S _ {\tau} = s) \pi (a | s, z _ {\tau}) P _ {\mathrm {e n v}} (s ^ {\prime} | s, a) \\ \cdot | \pi (a ^ {\prime} | s ^ {\prime}, z _ {\tau}) - \pi (a ^ {\prime} | s ^ {\prime}, \theta_ {t - \tau_ {\mathrm {m i x}}}) | d z _ {\tau} \\ \leq \sum_ {s, a, s ^ {\prime}, a ^ {\prime}} \int P (S _ {\tau} = s) P (\theta_ {\tau} = z _ {\tau} | S _ {\tau} = s) \pi (a | s, z _ {\tau}) P _ {\text {e n v}} \left(s ^ {\prime} | s, a\right) L _ {\pi} \| z _ {\tau} - \theta_ {t - \tau_ {\text {m i x}}} \| d z _ {\tau} \\ = n _ {a} L _ {\pi} \sum_ {s} \int P (S _ {\tau} = s) P \left(\theta_ {\tau} = z _ {\tau} \mid S _ {\tau} = s\right) \| z _ {\tau} - \theta_ {t - \tau_ {\operatorname {m i x}}} \| d z _ {\tau} \\ = n _ {a} L _ {\pi} \mathbb {E} \left[ \| \theta_ {\tau} - \theta_ {t - \tau_ {\operatorname* {m i x}}} \| \right]. \\ \end{array}
$$
For  $I_{2}$
$$
\begin{array}{l} I _ {2} = \sum_ {s, a, s ^ {\prime}, a ^ {\prime}} \int P (S _ {\tau} = s) P (\theta_ {\tau} = z _ {\tau} | S _ {\tau} = s) [ \pi (a | s, z _ {\tau}) - \pi (a | s, \theta_ {t - \tau_ {\mathrm {m i x}}}) ] P _ {\mathrm {e n v}} (s ^ {\prime} | s, a) \\ \cdot \pi \left(a ^ {\prime} \mid s ^ {\prime}, \theta_ {t - \tau_ {\text {m i x}}}\right) d z _ {\tau} \\ \leq \sum_ {s, a, s ^ {\prime}, a ^ {\prime}} \int P (S _ {\tau} = s) P (\theta_ {\tau} = z _ {\tau} | S _ {\tau} = s) L _ {\pi} \| z _ {\tau} - \theta_ {t - \tau_ {\mathrm {m i x}}} \| P _ {\mathrm {e n v}} (s ^ {\prime} | s, a) \pi (a ^ {\prime} | s ^ {\prime}, \theta_ {t - \tau_ {\mathrm {m i x}}}) d z _ {\tau} \\ = n _ {a} L _ {\pi} \sum_ {s} \int P (S _ {\tau} = s) P (\theta_ {\tau} = z _ {\tau} | S _ {\tau} = s) \| z _ {\tau} - \theta_ {\tau - \tau_ {\mathrm {m i x}}} \| d z _ {\tau} \\ = n _ {a} L _ {\pi} \mathbb {E} \left[ \| \theta_ {\tau} - \theta_ {t - \tau_ {\mathrm {m i x}}} \| \right]. \\ \end{array}
$$
For  $I_{3}$
$$
\begin{array}{l} I _ {3} = \sum_ {s, a, s ^ {\prime}, a ^ {\prime}} \int \left| P (S _ {\tau} = s) - P (\tilde {S} _ {\tau} = s) \right| P (\theta_ {\tau} = z _ {\tau} | S _ {\tau} = s) \pi (a | s, \theta_ {t - \tau_ {\mathrm {m i x}}}) P _ {\mathrm {e n v}} (s ^ {\prime} | s, a) \\ \cdot \pi (a ^ {\prime} | s ^ {\prime}, \theta_ {t - \tau_ {\mathrm {m i x}}}) d z _ {\tau} \\ = \sum_ {s} \left| P \left(S _ {\tau} = s\right) - P \left(\tilde {S} _ {\tau} = s\right) \right| \\ = \left\| P \left(S _ {\tau} \in \cdot\right) - P \left(\tilde {S} _ {\tau} \in \cdot\right) \right\| _ {1}. \\ \end{array}
$$
$$
\left| \left| P \left(O _ {\tau} \in \cdot\right) - P \left(\tilde {O} _ {\tau} \in \cdot\right) \right| \right| _ {1} \leq 2 n _ {a} L _ {\pi} \mathbb {E} \left[ \left| \left| \theta_ {\tau} - \theta_ {t - \tau_ {\text {m i x}}} \right| \right| \right] + \left| \left| P \left(S _ {\tau} \in \cdot\right) - P \left(\tilde {S} _ {\tau} \in \cdot\right) \right| \right| _ {1},
$$
which finishes the proof for part  $b$ .
It is easy to check that part  $a$  implies the following:
$$
\left| \left| P \left(S _ {\tau} \in \cdot\right) - P \left(\tilde {S} _ {\tau} \in \cdot\right) \right| \right| _ {1} \leq \left| \left| P \left(O _ {\tau - 1} \in \cdot\right) - P \left(\tilde {O} _ {\tau - 1} \in \cdot\right) \right| \right| _ {1}.
$$
The above fact, along with the result from part  $b$ , tells us the following:
$$
\left. \right.\left|\left| P \left(O _ {\tau} \in \cdot\right) - P \left(\tilde {O} _ {\tau} \in \cdot\right)\right|\right| _ {1} \leq 2 n _ {a} L _ {\pi} \mathbb {E} \left[\left|\left| \theta_ {\tau} - \theta_ {t - \tau_ {\operatorname* {m i x}}} \right|\right|\right] + \left|\left| P \left(O _ {\tau - 1} \in \cdot\right) - P \left(\tilde {O} _ {\tau - 1} \in \cdot\right)\right|\right| _ {1}.
$$
Repeat the inequality above over  $t$  to  $t - \tau_{\mathrm{mix}}$  we have
$$
| | P (O _ {t} \in \cdot) - P (\tilde {O} _ {t} \in \cdot) | | _ {1} \leq 2 n _ {a} L _ {\pi} \sum_ {i = t - \tau_ {\mathrm {m i x}}} ^ {t} \mathbb {E} \left[ \| \theta_ {i} - \theta_ {t - \tau_ {\mathrm {m i x}}} \| \right].
$$
C.5 Smoothness of the State-Value Function
The following two lemmas show that the state value function is actually smooth with respect to  $\theta$ . The idea here is inspired by Olshevsky & Gharesifard (2022). The first lemma requires the following basic identity from Zwillinger (2018): for matrix  $A_{\theta}$ ,
$$
\frac {\partial \left(A _ {\theta} ^ {- 1}\right)}{\partial \theta_ {i}} = - A _ {\theta} ^ {- 1} \frac {\partial A _ {\theta}}{\partial \theta_ {i}} A _ {\theta} ^ {- 1}.
$$
Lemma C.11. For two vectors  $u, v \in R^n$  whose entries are bounded. Suppose
$$
q _ {\theta} = u ^ {T} (I - \gamma P _ {\theta}) ^ {- 1} v.
$$
Then, there exists a constant  $L_{q}$  such that
$$
\left| \left| \nabla q _ {\theta} \right| \right| \leq L _ {q}.
$$
Similarly, if
$$
q _ {\theta} ^ {\prime} = u ^ {T} \left(I - \gamma P _ {\theta} ^ {\prime}\right) ^ {- 1} v.
$$
Then, there exists a constant  $L_{q}^{\prime}$  such that
$$
\left| \left| \nabla q _ {\theta} ^ {\prime} \right| \right| \leq L _ {q} ^ {\prime}.
$$
The following lemma, which is our goal in this section, claims that  $V_{\theta}^{*}$  is smooth with respect to  $\theta$ .
Lemma C.12.  $V_{\theta_t}^*$  is  $H_V$ -smoothness with respect to  $\theta_t$ .
Proof. Using Theorem 2.1, we obtain the following result:
$$
\begin{array}{l} \left| \left| \nabla V _ {\theta_ {1}} ^ {*} - \nabla V _ {\theta_ {2}} ^ {*} \right| \right| \\ = \left\| \sum_ {(s, a) \in S \times A} \phi_ {\theta_ {1}} (s, a) Q _ {\theta_ {1}} ^ {*} (s, a) \nabla \ln \pi (a | s, \theta_ {1}) - \sum_ {(s, a) \in S \times A} \phi_ {\theta_ {2}} (s, a) Q _ {\theta_ {2}} ^ {*} (s, a) \nabla \ln \pi (a | s, \theta_ {2}) \right\| \\ \leq \left\| \sum_ {(s, a) \in S \times A} \phi_ {\theta_ {1}} (s, a) Q _ {\theta_ {1}} ^ {*} (s, a) \nabla \ln \pi (a | s, \theta_ {1}) - \sum_ {(s, a) \in S \times A} \phi_ {\theta_ {1}} (s, a) Q _ {\theta_ {1}} ^ {*} (s, a) \nabla \ln \pi (a | s, \theta_ {2}) \right\| \\ + \left\| \sum_ {(s, a) \in S \times A} \phi_ {\theta_ {1}} (s, a) Q _ {\theta_ {1}} ^ {*} (s, a) \nabla \ln \pi (a | s, \theta_ {2}) - \sum_ {(s, a) \in S \times A} \phi_ {\theta_ {1}} (s, a) Q _ {\theta_ {2}} ^ {*} (s, a) \nabla \ln \pi (a | s, \theta_ {2}) \right\| \\ + \left\| \sum_ {(s, a) \in S \times A} \phi_ {\theta_ {1}} (s, a) Q _ {\theta_ {2}} ^ {*} (s, a) \nabla \ln \pi (a | s, \theta_ {2}) - \sum_ {(s, a) \in S \times A} \phi_ {\theta_ {2}} (s, a) Q _ {\theta_ {2}} ^ {*} (s, a) \nabla \ln \pi (a | s, \theta_ {2}) \right\| \\ \leq \underbrace {\sum_ {(s , a) \in S \times A} \phi_ {\theta_ {1}} (s , a) Q _ {\theta_ {1}} ^ {*} (s , a) \| \nabla \ln \pi (a | s , \theta_ {1}) - \nabla \ln \pi (a | s , \theta_ {2}) \|} _ {I _ {1}} \\ + \underbrace {\sum_ {(s , a) \in S \times A} \phi_ {\theta_ {1}} (s , a) \left| Q _ {\theta_ {1}} ^ {*} (s , a) - Q _ {\theta_ {2}} ^ {*} (s , a) \right| \nabla \ln \pi (a | s , \theta_ {2})} _ {I _ {2}} \\ + \underbrace {\sum_ {(s , a) \in S \times A} | \phi_ {\theta_ {1}} (s , a) - \phi_ {\theta_ {2}} (s , a) | Q _ {\theta_ {2}} ^ {*} (s , a) \nabla \ln \pi (a | s , \theta_ {2})} _ {I _ {3}}. \\ \end{array}
$$
We now show that all  $I_1, I_2, I_3$  can be bounded by a multiple of  $\|\theta_1 - \theta_2\|$ .
For  $I_{1}$ , by Assumption 2.4 we know that  $\nabla \ln \pi (a|s,\theta)$  is Lipschitz, which, together with  $\phi_{\theta}(s,a)\leq \frac{1}{1 - \gamma}$  and  $Q_{\theta}^{*}(s,a)\leq \frac{r_{\max}}{1 - \gamma}$ , implies that  $I_{1}$  can be upper bounded by a multiple of  $||\theta_1 - \theta_2||$
For  $I_{2}$ , since  $Q_{\theta}^{*}$  satisfies Bellman equation, we can write  $Q_{\theta}^{*}$  using matrix multiplication, which is
$$
Q _ {\theta} ^ {*} = (1 - \gamma P _ {\theta}) ^ {- 1} R.
$$
By Lemma C.11, this implies
$$
Q _ {\theta} ^ {*} (s, a) = e _ {s, a} ^ {T} (1 - \gamma P _ {\theta}) ^ {- 1} R,
$$
where  $e_{s,a}$  has only one non-zero entry of one corresponding to the pair  $(s,a)$ . Hence,  $Q_{\theta}^{*}(s,a)$  is Lipschitz with respect to  $\theta$ .
For  $I_3$ , by definition,
$$
P _ {\theta} (S _ {t} = s | S _ {0} = s _ {0}) = e _ {s _ {0}} ^ {T} P _ {\theta} ^ {\prime t} e _ {s}.
$$
$$
\phi_ {\theta} (s, a) = \phi_ {\theta} (s) \pi (a | s, \theta) = e _ {s _ {0}} ^ {T} (I - \gamma P _ {\theta} ^ {\prime}) ^ {- 1} e _ {s} \pi (a | s, \theta),
$$
where  $e_s$  has only one non-zero entry of one corresponding to  $s$ . Again, by Lemma C.11,  $\phi_{\theta}(s,a)$  is Lipschitz with respect to  $\theta$ .
Proof. The proof can be found in Olshevsky & Gharesifard (2022).
C.6 Properties of the Actor Update
The following lemma shows that the incremental in actor update is bounded.
Lemma C.13. For  $g(\hat{O}_t, w_t, \theta_t)$  and  $\bar{g}(w_t, \theta_t)$ , we have the following properties:
a. For  $g(\hat{O}_t, w_t, \theta_t)$ ,
$$
\left| \left| g \left(\hat {O} _ {t}, w _ {t}, \theta_ {t}\right) \right| \right| \leq U _ {g}
$$
$$
w h e r e U _ {g} = 2 L _ {\pi} ^ {\prime} L _ {Q} \sigma_ {w} + L _ {\pi} ^ {\prime} \epsilon + L _ {\pi} ^ {\prime} \frac {r _ {\max}}{1 - \gamma}.
$$
b. For  $\bar{g} (w_t,\theta_t)$
$$
| | \bar {g} (w _ {t}, \theta_ {t}) | | \leq \frac {1}{1 - \gamma} U _ {g}
$$
$$
w h e r e U _ {g} = 2 L _ {\pi} ^ {\prime} L _ {Q} \sigma_ {w} + L _ {\pi} ^ {\prime} \epsilon + L _ {\pi} ^ {\prime} \frac {r _ {\max }}{1 - \gamma}.
$$
Proof. Recall that in Eq.(15),  $g(\hat{O}_t, w_t, \theta_t)$  is defined to be
$$
g (\hat {O} _ {t}, w _ {t}, \theta_ {t}) = Q (\hat {s} _ {t}, \hat {a} _ {t}, w _ {t}) \nabla_ {\theta} \ln \pi (\hat {a} _ {t} | \hat {s} _ {t}, \theta_ {t}).
$$
To bound  $||g(\hat{O}_t, w_t, \theta_t)||$ , by Assumption 2.4 and Lemma C.1, we can do the following manipulations
$$
\begin{array}{l} \left. \right.\left\| g \left(\hat {O} _ {t}, w _ {t}, \theta_ {t}\right)\right\| = \left\| \nabla_ {\theta} \ln \pi \left(\hat {a} _ {t} \mid \hat {s} _ {t}, \theta_ {t}\right)\left(Q \left(\hat {s} _ {t}, \hat {a} _ {t}, w _ {t}\right) - Q \left(\hat {s} _ {t}, \hat {a} _ {t}, \hat {w} _ {\theta_ {t}} ^ {*}\right) + Q \left(\hat {s} _ {t}, \hat {a} _ {t}, \hat {w} _ {\theta_ {t}} ^ {*}\right)\right)\right\| \\ \leq \left\| \nabla_ {\theta} \ln \pi (\hat {a} _ {t} | \hat {s} _ {t}, \theta_ {t}) \left(Q (\hat {s} _ {t}, \hat {a} _ {t}, w _ {t}) - Q (\hat {s} _ {t}, \hat {a} _ {t}, \hat {w} _ {\theta_ {t}} ^ {*})\right) \right\| \\ + \left\| \nabla_ {\theta} \ln \pi \left(\hat {a} _ {t} | \hat {s} _ {t}, \theta_ {t}\right) Q \left(\hat {s} _ {t}, \hat {a} _ {t}, \hat {w} _ {\theta_ {t} ^ {*}} ^ {*}\right) \right\| \\ \leq L _ {\pi} ^ {\prime} L _ {Q} \left| \left| w _ {t} - \hat {w} _ {\theta_ {t}} ^ {*} \right| \right| + \left\| \nabla_ {\theta} \ln \pi \left(\hat {a} _ {t} | \hat {s} _ {t}, \theta_ {t}\right) Q \left(\hat {s} _ {t}, \hat {a} _ {t}, \hat {w} _ {\theta_ {t}} ^ {*}\right) \right\| \\ \leq 2 L _ {\pi} ^ {\prime} L _ {Q} \sigma_ {w} + \left\| \nabla_ {\theta} \ln \pi (\hat {a} _ {t} | \hat {s} _ {t}, \theta_ {t}) \left(Q (\hat {s} _ {t}, \hat {a} _ {t}, \hat {w} _ {\theta_ {t}} ^ {*}) - Q ^ {*} (\hat {s} _ {t}, \hat {a} _ {t}) + Q ^ {*} (\hat {s} _ {t}, \hat {a} _ {t})\right) \right\| \\ = 2 L _ {\pi} ^ {\prime} L _ {Q} \sigma_ {w} + \left\| \nabla_ {\theta} \ln \pi \left(\hat {a} _ {t} | \hat {s} _ {t}, \theta_ {t}\right) \left(Q \left(\hat {s} _ {t}, \hat {a} _ {t}, \hat {w} _ {\theta_ {t}} ^ {*}\right) - Q ^ {*} (\hat {s} _ {t}, \hat {a} _ {t})\right) \right\| \\ + \left\| \nabla_ {\theta} \ln \pi \left(\hat {a} _ {t} \mid \hat {s} _ {t}, \theta_ {t}\right) Q ^ {*} \left(\hat {s} _ {t}, \hat {a} _ {t}\right) \right\| \\ \leq 2 L _ {\pi} ^ {\prime} L _ {Q} \sigma_ {w} + L _ {\pi} ^ {\prime} \epsilon + L _ {\pi} ^ {\prime} \frac {r _ {\mathrm {m a x}}}{1 - \gamma}. \\ \end{array}
$$
Because Eq.(16) implies that  $\bar{g} (w_t,\theta_t)$  is some expectation of  $g(\hat{O}_t,w_t,\theta_t)$  with a coefficient  $\frac{1}{1 - \gamma}$ , the second part of this lemma is a direct result of the first part in Lemma C.13.
C.7 Properties of the Critic Update
In this section, we will introduce some properties that we find useful in analyzing critic update.
Notice that, as defined in Eq.(8),  $\delta_t$  actually depends on  $O_t$  and  $w_t$ . Here we make this dependency explicitly and thus write  $\delta_t$  as  $\delta_t = \delta(O_t, w_t)$ . In this following lemma, we explore some properties of  $\delta_t$ .
Lemma C.14. For  $\delta_t$ , we have the following two results:
a.  $\delta (O_t,w)$  is  $L_{\delta}$ -Lipschitz with respect to  $w$
$$
\left| \delta \left(O _ {t}, w _ {1}\right) - \delta \left(O _ {t}, w _ {2}\right) \right| \leq L _ {\delta} \left\| w _ {1} - w _ {2} \right\|,
$$
$$
w h e r e L _ {\delta} = (1 + \gamma) L _ {Q}.
$$
b.  $|\delta (O_t,w)|$  is upper bounded by  $U_{\delta}$
$$
\left| \delta \left(O _ {t}, w\right) \right| \leq U _ {\delta},
$$
$$
w h e r e U _ {\delta} = 2 (1 + \gamma) L _ {Q} \sigma_ {w} + (1 + \gamma) \epsilon + \frac {2}{1 - \gamma} r _ {\max }
$$
Proof. Using Eq.(8),
$$
\begin{array}{l} \left| \delta \left(O _ {t}, w _ {1}\right) - \delta \left(O _ {t}, w _ {2}\right) \right| \\ = \left| \left(r \left(s _ {t}, a _ {t}\right) + \gamma Q \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}, w _ {1}\right) - Q \left(s _ {t}, a _ {t}, w _ {1}\right)\right) - \left(r \left(s _ {t}, a _ {t}\right) + \gamma Q \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}, w _ {2}\right) - Q \left(s _ {t}, a _ {t}, w _ {2}\right)\right) \right| \\ = \left| \gamma \left(Q \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}, w _ {1}\right) - Q \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}, w _ {2}\right)\right) - \left(Q \left(s _ {t}, a _ {t}, w _ {1}\right) - Q \left(s _ {t}, a _ {t}, w _ {2}\right)\right) \right| \\ \leq (1 + \gamma) L _ {Q} | | w _ {1} - w _ {2} | |, \\ \end{array}
$$
where the last line uses Lemma C.1. On the other hand,
$$
\left| \delta \left(O _ {t}, w\right) \right| = \left| r \left(s _ {t}, a _ {t}\right) + \gamma Q \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}, w\right) - Q \left(s _ {t}, a _ {t}, w\right) \right|
$$
$$
\begin{array}{l} = \left| r \left(s _ {t}, a _ {t}\right) + \gamma \left(Q \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}, w\right) - \hat {Q} _ {\theta} ^ {*} \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}\right)\right) - \left(Q \left(s _ {t}, a _ {t}, w\right) - \hat {Q} _ {\theta} ^ {*} \left(s _ {t}, a _ {t}\right)\right) \right. \\ + \gamma \left(\hat {Q} _ {\theta} ^ {*} \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}\right) - Q _ {\theta} ^ {*} \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}\right)\right) - \left(\hat {Q} _ {\theta} ^ {*} \left(s _ {t}, a _ {t}\right) - Q _ {\theta} ^ {*} \left(s _ {t}, a _ {t}\right)\right) + \gamma Q _ {\theta} ^ {*} \left(s _ {t} ^ {\prime}, a _ {t} ^ {\prime}\right) - Q _ {\theta} ^ {*} \left(s _ {t}, a _ {t}\right) \\ \end{array}
$$
$$
\begin{array}{l} \leq r _ {\max } + (1 + \gamma) L _ {Q} | | w - \hat {w} _ {\theta} ^ {*} | | + (1 + \gamma) \epsilon + \frac {1 + \gamma}{1 - \gamma} r _ {\max } \\ \leq 2 (1 + \gamma) L _ {Q} \sigma_ {w} + (1 + \gamma) \epsilon + \frac {2}{1 - \gamma} r _ {\max}, \\ \end{array}
$$
where the second equation uses Assumption 2.1, Lemma C.1, Eq.(9) and Eq.(5).
With the above properties of  $\delta_t$ , now we can further consider  $f, \bar{f}$ , and  $F$  defined in Lemma C.17.
Lemma C.15. For  $f(O_{t},w)$  defined in Eq.(15), we have the following two results:
a.  $f(O_{t},w)$  is  $L_{f}$ -Lipschitz with respect to  $w$ ,
$$
\left| \left| f \left(O _ {t}, w _ {1}\right) - f \left(O _ {t}, w _ {2}\right) \right| \right| \leq L _ {f} \left| \left| w _ {1} - w _ {2} \right| \right|,
$$
where  $L_{f} = (1 + \gamma)L_{Q}^{2} + H_{Q}U_{\delta}$
b.  $||f(O_t,w)||$  can be upper bounded by  $U_{f}$
$$
\left\| f \left(O _ {t}, w\right) \right\| \leq U _ {f},
$$
where  $U_{f} = L_{Q}U_{\delta}$
$$
f (O _ {t}, w) = \delta (O _ {t}, w) \nabla Q (s _ {t}, a _ {t}, w).
$$
$$
\begin{array}{l} f (O _ {t}, w _ {1}) - f (O _ {t}, w _ {2}) \\ = \delta \left(O _ {t}, w _ {1}\right) \nabla Q \left(s _ {t}, a _ {t}, w _ {1}\right) - \delta \left(O _ {t}, w _ {2}\right) \nabla Q \left(s _ {t}, a _ {t}, w _ {2}\right) \\ = \delta \left(O _ {t}, w _ {1}\right) \nabla Q \left(s _ {t}, a _ {t}, w _ {1}\right) - \delta \left(O _ {t}, w _ {2}\right) \nabla Q \left(s _ {t}, a _ {t}, w _ {1}\right) \\ + \delta (O _ {t}, w _ {2}) \nabla Q \left(s _ {t}, a _ {t}, w _ {1}\right) - \delta (O _ {t}, w _ {2}) \nabla Q \left(s _ {t}, a _ {t}, w _ {2}\right) \\ = \underbrace {\left[ \delta \left(O _ {t} , w _ {1}\right) - \delta \left(O _ {t} , w _ {2}\right) \right] \nabla Q \left(s _ {t} , a _ {t} , w _ {1}\right)} _ {I _ {1}} + \underbrace {\delta \left(O _ {t} , w _ {2}\right) \left(\nabla Q \left(s _ {t} , a _ {t} , w _ {1}\right) - \nabla Q \left(s _ {t} , a _ {t} , w _ {2}\right)\right)} _ {I _ {2}}. \\ \end{array}
$$
For  $I_{1}$ , by Lemma C.14 and Lemma C.1, we perform the following manipulations:
$$
\begin{array}{l} \left\| \left[ \delta \left(O _ {t}, w _ {1}\right) - \delta \left(O _ {t}, w _ {2}\right) \right] \nabla Q \left(s _ {t}, a _ {t}, w _ {1}\right) \right\| \leq L _ {\delta} L _ {Q} | | w _ {1} - w _ {2} | | \\ = (1 + \gamma) L _ {Q} ^ {2} | | w _ {1} - w _ {2} | |. \\ \end{array}
$$
For  $I_{2}$ , by Lemma C.14 and Lemma C.1, we derive
$$
\left\| \delta \left(O _ {t}, w _ {2}\right) \left(\nabla Q \left(s _ {t}, a _ {t}, w _ {1}\right) - \nabla Q \left(s _ {t}, a _ {t}, w _ {2}\right)\right) \right\| \leq U _ {\delta} H _ {Q} \| w _ {1} - w _ {2} \|.
$$
Combining  $I_{1}$  and  $I_{2}$  we prove the first part of this lemma.
For the second part, by Lemma C.14 and Lemma C.1,
$$
\left| \left| f \left(O _ {t}, w\right) \right| \right| = \left| \left| \delta \left(O _ {t}, w\right) \nabla Q \left(s _ {t}, a _ {t}, w\right) \right| \right| \leq U _ {\delta} L _ {Q}.
$$
Lemma C.16. For  $\bar{f} (w,\theta)$  defined in Eq.(16), we have the following results:
a.  $||\bar{f} (w,\theta)||$  can be upper bounded by  $U_{f}$
$$
| | \bar {f} (w, \theta) | | \leq U _ {f},
$$
b.  $\bar{f} (w,\theta)$  is  $L_{f}$ -Lipschitz with respect to  $w$
$$
| | \bar {f} (w _ {1}, \theta) - \bar {f} (w _ {2}, \theta) | | \leq L _ {\bar {f}} | | w _ {1} - w _ {2} | |,
$$
c.  $\bar{f} (w,\theta)$  is  $L_{\bar{f}}$  -Lipschitz with respect to  $\theta$
$$
\left| \left| \bar {f} (w, \theta_ {1}) - \bar {f} (w, \theta_ {2}) \right| \right| \leq L _ {\bar {f}} \left| \left| \theta_ {1} - \theta_ {2} \right| \right|,
$$
where  $L_{\bar{f}} = L_QU_\delta (2 + \tau_{\mathrm{mix}} + C\frac{1}{1 - \beta})n_aL_\pi$
Proof. Because of Eq.(16), the first and second part of this lemma is a direct result of Lemma C.15. For the third part,