SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \frac {\mathrm {d}}{\mathrm {d} \theta} = \sum_ {i \geq 0} \frac {\mathrm {d} a _ {i}}{\mathrm {d} \theta} \cdot \frac {\partial}{\partial a _ {i}} + \sum_ {i > 0} \frac {\mathrm {d} s _ {i}}{\mathrm {d} \theta} \cdot \frac {\partial}{\partial s _ {i}} \\ = \frac {\mathrm {d} a _ {0}}{\mathrm {d} \theta} \cdot \frac {\partial}{\partial a _ {0}} + \frac {\mathrm {d} s _ {1}}{\mathrm {d} \theta} \cdot \frac {\partial}{\partial s _ {1}} + \sum_ {i \geq 1} \frac {\mathrm {d} a _ {i}}{\mathrm {d} \theta} \cdot \frac {\partial}{\partial a _ {i}} + \sum_ {i > 1} \frac {\mathrm {d} s _ {i}}{\mathrm {d} \theta} \cdot \frac {\partial}{\partial s _ {i}}. \tag {A.2} \\ \end{array}
$$
Here, the expansion holds when  $\mathrm{d} / \mathrm{d}\theta$  operates on policies and models that are differentiable with respect to all states  $s_i$  and actions  $a_i$ .
Plugging (A.2) into (A.1), we obtain the following recursive formula for  $\nabla_{\theta}^{i}$
$$
\nabla_ {\theta} ^ {i} = \frac {\mathrm {d} a _ {i}}{\mathrm {d} \theta} \cdot \frac {\partial}{\partial a _ {i}} + \frac {\mathrm {d} a _ {t}}{\mathrm {d} \theta} \cdot \frac {\mathrm {d} s _ {i + 1}}{\mathrm {d} a _ {t}} \cdot \frac {\partial}{\partial s _ {i + 1}} + \nabla_ {\theta} ^ {i + 1}. \tag {A.3}
$$
By the Bellman equation, we have
$$
V ^ {\pi_ {\theta}} (s) = \mathbb {E} _ {\varsigma} \left[ (1 - \gamma) \cdot r (s, \pi_ {\theta} (s, \varsigma)) + \gamma \cdot \mathbb {E} _ {\xi^ {*}} \left[ V ^ {\pi_ {\theta}} \left(f (s, \pi_ {\theta} (s, \varsigma), \xi^ {*})\right) \right] \right]. \tag {A.4}
$$
Combining (A.3) and (A.4) gives
$$
\begin{array}{l} \nabla_ {\theta} V ^ {\pi_ {\theta}} (s) = \frac {\mathrm {d} V ^ {\pi_ {\theta}} (s)}{\mathrm {d} \theta} = \frac {\mathrm {d}}{\mathrm {d} \theta} \mathbb {E} _ {\varsigma} \left[ (1 - \gamma) \cdot r (s, \pi_ {\theta} (s, \varsigma)) + \gamma \cdot \mathbb {E} _ {\xi^ {*}} \left[ V ^ {\pi_ {\theta}} \left(f (s, \pi_ {\theta} (s, \varsigma), \xi^ {*})\right) \right] \right] \\ = \mathbb {E} _ {\varsigma} \left[ (1 - \gamma) \cdot \frac {\partial r}{\partial a} \cdot \frac {\mathrm {d} a}{\mathrm {d} \theta} + \gamma \cdot \mathbb {E} _ {\xi^ {*}} \left[ \frac {\mathrm {d} a}{\mathrm {d} \theta} \cdot \frac {\partial s ^ {\prime}}{\partial a} \cdot \frac {\mathrm {d} V ^ {\pi_ {\theta}} \left(s ^ {\prime}\right)}{\mathrm {d} s ^ {\prime}} + \frac {\mathrm {d} V ^ {\pi_ {\theta}} \left(s ^ {\prime}\right)}{\mathrm {d} \theta} \right] \right], \tag {A.5} \\ \end{array}
$$
which corresponds to (3.2).
For the  $\mathrm{d}V^{\pi}(s) / \mathrm{d}s$  term on the right-hand side of (A.5), we have the following recursion,
$$
\begin{array}{l} \nabla_ {s} V ^ {\pi_ {\theta}} (s) = \frac {\mathrm {d} V ^ {\pi_ {\theta}} (s)}{\mathrm {d} s} = \frac {\mathrm {d}}{\mathrm {d} s} \mathbb {E} _ {\varsigma} \left[ (1 - \gamma) \cdot r (s, \pi_ {\theta} (s, \varsigma)) + \gamma \cdot \mathbb {E} _ {\xi^ {*}} \left[ V ^ {\pi_ {\theta}} \left(f (s, \pi_ {\theta} (s, \varsigma), \xi^ {*})\right) \right] \right] \\ = \mathbb {E} _ {\varsigma} \left[ (1 - \gamma) \cdot \left(\frac {\partial r}{\partial s} + \frac {\partial r}{\partial a} \cdot \frac {\partial a}{\partial s}\right) + \gamma \cdot \mathbb {E} _ {\xi^ {*}} \left[ \frac {\partial s ^ {\prime}}{\partial s} \cdot \frac {\mathrm {d} V ^ {\pi_ {\theta}} \left(s ^ {\prime}\right)}{\mathrm {d} s ^ {\prime}} + \frac {\partial s ^ {\prime}}{\partial a} \cdot \frac {\partial a}{\partial s} \cdot \frac {\mathrm {d} V ^ {\pi_ {\theta}} \left(s ^ {\prime}\right)}{\mathrm {d} s ^ {\prime}} \right] \right], \tag {A.6} \\ \end{array}
$$
which corresponds to (3.3).
Therefore, we complete the derivative of (3.2) and (3.3).
Derivation of RP-DR Policy Gradient. By the same arguments in (A.5) and (A.6) with  $\nabla_{a}f$  (or  $\partial s' / \partial a$ ) and  $\nabla_{s}f$  (or  $\partial s' / \partial s$ ) replaced with  $\nabla_{a}\widehat{f}_{\psi}$  and  $\nabla_{s}\widehat{f}_{\psi}$ , we obtain
$$
\begin{array}{l} \widehat {\nabla} _ {\theta} V ^ {\pi_ {\theta}} (\widehat {s} _ {i, n}) = (1 - \gamma) \nabla_ {a} r (\widehat {s} _ {i, n}, \widehat {a} _ {i, n}) \nabla_ {\theta} \pi_ {\theta} (\widehat {s} _ {i, n}, \varsigma_ {n}) \\ + \gamma \widehat {\nabla} _ {s} V ^ {\pi_ {\theta}} \left(\widehat {s} _ {i + 1, n}\right) \nabla_ {a} \widehat {f} _ {\psi} \left(\widehat {s} _ {i, n}, \widehat {a} _ {i, n}, \xi_ {n}\right) \nabla_ {\theta} \pi_ {\theta} \left(\widehat {s} _ {i, n}, \varsigma_ {n}\right) + \gamma \widehat {\nabla} _ {\theta} V ^ {\pi_ {\theta}} \left(\widehat {s} _ {i + 1, n}\right), \tag {A.7} \\ \end{array}
$$
$$
\begin{array}{l} \widehat {\nabla} _ {s} V ^ {\pi_ {\theta}} \left(\widehat {s} _ {i, n}\right) = (1 - \gamma) \left(\nabla_ {s} r \left(\widehat {s} _ {i, n}, \widehat {a} _ {i, n}\right) + \nabla_ {a} r \left(\widehat {s} _ {i, n}, \widehat {a} _ {i, n}\right) \nabla_ {s} \pi_ {\theta} \left(\widehat {s} _ {i, n}, \varsigma_ {n}\right)\right) \\ + \gamma \widehat {\nabla} _ {s} V ^ {\pi_ {\theta}} \left(\widehat {s} _ {i + 1, n}\right) \left(\nabla_ {s} \widehat {f} _ {\psi} \left(\widehat {s} _ {i, n}, \widehat {a} _ {i, n}, \xi_ {n}\right) + \nabla_ {a} \widehat {f} _ {\psi} \left(\widehat {s} _ {i, n}, \widehat {a} _ {i, n}, \xi_ {n}\right) \nabla_ {s} \pi_ {\theta} \left(\widehat {s} _ {i, n}, \varsigma_ {n}\right)\right), \tag {A.8} \\ \end{array}
$$
where the termination of backpropagation at the  $h$ -th step is  $\widehat{\nabla} V^{\pi_{\theta}}(\widehat{s}_{h,n}) = \nabla \widehat{V}_{\omega}(\widehat{s}_{h,n})$ .
Combining (A.7) and (A.8), we obtain the RP-DR policy gradient estimator as follows,
$$
\widehat {\nabla} _ {\theta} ^ {\mathrm {D R}} J (\pi_ {\theta}) = \frac {1}{N} \sum_ {n = 1} ^ {N} \widehat {\nabla} _ {\theta} V ^ {\pi_ {\theta}} \left(\widehat {s} _ {0, n}\right) = \frac {1}{N} \sum_ {n = 1} ^ {N} \nabla_ {\theta} \left(\sum_ {i = 0} ^ {h - 1} \gamma^ {i} r \left(\widehat {s} _ {i, n}, \widehat {a} _ {i, n}\right) + \gamma^ {h} \widehat {Q} _ {\omega} \left(\widehat {s} _ {h, n}, \widehat {a} _ {h, n}\right)\right), \tag {A.9}
$$
where  $\widehat{s}_{0,n} \sim \mu_{\pi_\theta}$ ,  $\widehat{a}_{i,n} = \pi_\theta(\widehat{s}_{i,n}, \varsigma_n)$ , and  $\widehat{s}_{i+1,n} = \widehat{f}_\psi(\widehat{s}_{i,n}, \widehat{a}_{i,n}, \xi_n)$ . Here,  $\varsigma_n$  and  $\xi_n$  are inferred by solving  $a_i = \pi_\theta(s_i, \varsigma_n)$  and  $s_{i+1} = \widehat{f}_\psi(s_i, a_i, \xi_n)$ , respectively, where  $(s_i, a_i, s_{i+1})$  is the real data sample. For example, for a state  $s_{i+1}$  sampled from a one-dimensional Gaussian transition model  $s_{i+1} \sim \mathcal{N}(\phi(s_i, a_i), \sigma^2)$ , where the variance is  $\sigma$  and the mean  $\phi(s_i, a_i)$  is the output of some function parameterized by  $\phi$ , the noise  $\xi_n$  can be inferred as  $\xi_n = (s_{i+1} - \phi(s_i, a_i)) / \sigma$ .
B.1 Proof of Proposition 5.2
As a preparation before proving Proposition 5.2, we first present the following lemma stating that the objective in (2.1) is Lipschitz smooth under Assumption 5.1.
Lemma B.1 (Smooth Objective, [69] Lemma 3.2). The objective  $J(\pi_{\theta})$  is  $L$ -smooth in  $\theta$ , such that  $\| \nabla_{\theta}J(\pi_{\theta_1}) - \nabla_{\theta}J(\pi_{\theta_2})\| _2\leq L\| \theta_1 - \theta_2\| _2$ , where
$$
L = \frac {r _ {\mathrm {m}} \cdot L _ {1}}{(1 - \gamma) ^ {2}} + \frac {(1 + \gamma) \cdot r _ {\mathrm {m}} \cdot B _ {\theta} ^ {2}}{(1 - \gamma) ^ {3}}.
$$
Then we are ready to prove Proposition 5.2.
Proof of Proposition 5.2. See the proof of Theorem 4.2 in [71].
B.2 Proof of Proposition 5.4
Proof. Since the RP-DP gradient  $\widehat{\nabla}_{\theta}J(\pi_{\theta}) = \widehat{\nabla}_{\theta}^{\mathrm{DP}}J(\pi_{\theta})$  in (4.1) and the RP-DR gradient  $\widehat{\nabla}_{\theta}J(\pi_{\theta}) = \widehat{\nabla}_{\theta}^{\mathrm{DR}}J(\pi_{\theta})$  in (4.2) share the same state transition  $\widehat{s}_{i+1,n} = \widehat{f}(\widehat{s}_{i,n}, \xi_n)$ , where recall that the only difference lies in the source of noise  $\xi_n$ , our subsequent analysis holds for both RP-DP and RP-DR.
To upper-bound the gradient variance  $v_{t} = \mathbb{E}[||\widehat{\nabla}_{\theta}J(\pi_{\theta_{t}}) - \mathbb{E}[\widehat{\nabla}_{\theta}J(\pi_{\theta_{t}})]||_{2}^{2}]$ , we characterize the norm inside the outer expectation.
We start with the case where the sample size  $N = 1$ , which naturally generalizes to  $N > 1$ . Specifically, we consider an arbitrary  $h$ -step trajectory obtained by unrolling the model under policy  $\pi_{\theta_t}$ . We denote the pathwise gradient  $\widehat{\nabla}_{\theta}J(\pi_{\theta_t})$  of this trajectory as  $g'$ . Then we have
$$
v _ {t} \leq \max _ {g ^ {\prime}} \left\| g ^ {\prime} - \mathbb {E} \big [ \widehat {\nabla} _ {\theta} J (\pi_ {\theta_ {t}}) \big ] \right\| _ {2} ^ {2} = \left\| g - \mathbb {E} \big [ \widehat {\nabla} _ {\theta} J (\pi_ {\theta_ {t}}) \big ] \right\| _ {2} ^ {2} = \left\| \mathbb {E} \big [ g - \widehat {\nabla} _ {\theta} J (\pi_ {\theta_ {t}}) \big ] \right\| _ {2} ^ {2},
$$
where  $g$  is the pathwise gradient  $\widehat{\nabla}_{\theta}J(\pi_{\theta_t})$  of a fixed (but unknown) trajectory  $(\widehat{s}_{0,n}, \widehat{a}_{0,n}, \widehat{s}_{1,n}, \widehat{a}_{1,n}, \dots)$  such that the maximum is achieved.
Using the fact that  $\| \mathbb{E}[\cdot ]\| _2\leq \mathbb{E}[\| \cdot \| _2]$ , we further obtain
$$
v _ {t} \leq \mathbb {E} \left[ \left\| g - \widehat {\nabla} _ {\theta} J \left(\pi_ {\theta_ {t}}\right) \right\| _ {2} \right] ^ {2}. \tag {B.1}
$$
Let  $\widehat{x}_{i,n} = (\widehat{s}_{i,n},\widehat{a}_{i,n})$ . By the triangular inequality, we have
$$
\begin{array}{l} \mathbb {E} \left[ \left\| g - \widehat {\nabla} _ {\theta} J (\pi_ {\theta_ {t}}) \right\| _ {2} \right] \leq \sum_ {i = 0} ^ {h - 1} \gamma^ {i} \cdot \mathbb {E} _ {\overline {{x}} _ {i}} \left[ \left\| \nabla_ {\theta} r (\widehat {x} _ {i, n}) - \nabla_ {\theta} r (\overline {{x}} _ {i}) \right\| _ {2} \right] \\ + \gamma^ {h} \cdot \mathbb {E} _ {\bar {x} _ {h}} \left[ \left\| \nabla \widehat {Q} (\widehat {x} _ {h, n}) \nabla_ {\theta} \widehat {x} _ {h, n} - \nabla \widehat {Q} (\bar {x} _ {h}) \nabla_ {\theta} \bar {x} _ {h} \right\| _ {2} \right]. \tag {B.2} \\ \end{array}
$$
By the chain rule, we have for any  $i \geq 1$  that
$$
\frac {\mathrm {d} \widehat {a} _ {i , n}}{\mathrm {d} \theta} = \frac {\partial \widehat {a} _ {i , n}}{\partial \widehat {s} _ {i , n}} \cdot \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} + \frac {\partial \widehat {a} _ {i , n}}{\partial \theta}, \tag {B.3}
$$
$$
\frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} = \frac {\partial \widehat {s} _ {i , n}}{\partial \widehat {s} _ {i - 1 , n}} \cdot \frac {\mathrm {d} \widehat {s} _ {i - 1 , n}}{\mathrm {d} \theta} + \frac {\partial \widehat {s} _ {i , n}}{\partial \widehat {a} _ {i - 1 , n}} \cdot \frac {\mathrm {d} \widehat {a} _ {i - 1 , n}}{\mathrm {d} \theta}. \tag {B.4}
$$
Denote by  $L_{\theta} = \sup_{\theta \in \Theta ,s\in S,\varsigma}\| \nabla_{\theta}\pi_{\theta}(s,\varsigma)\| _2$
Plugging  $\mathrm{d}\widehat{a}_{i - 1,n} / \mathrm{d}\theta$  in (B.3) into (B.4), we get
$$
\begin{array}{l} \left\| \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} \right\| _ {2} = \left\| \left(\frac {\partial \widehat {s} _ {i , n}}{\partial \widehat {s} _ {i - 1 , n}} + \frac {\partial \widehat {s} _ {i , n}}{\partial \widehat {a} _ {i - 1 , n}} \cdot \frac {\partial \widehat {a} _ {i - 1 , n}}{\partial \widehat {s} _ {i - 1 , n}}\right) \cdot \frac {\mathrm {d} \widehat {s} _ {i - 1 , n}}{\mathrm {d} \theta} + \frac {\partial \widehat {s} _ {i , n}}{\partial \widehat {a} _ {i - 1 , n}} \cdot \frac {\partial \widehat {a} _ {i - 1 , n}}{\partial \theta} \right\| _ {2} \\ \leq L _ {\widehat {f}} \widetilde {L} _ {\pi} \cdot \left\| \frac {\mathrm {d} \widehat {s} _ {i - 1 , n}}{\mathrm {d} \theta} \right\| _ {2} + L _ {\widehat {f}} L _ {\theta}, \tag {B.5} \\ \end{array}
$$
where the inequality follows from Assumption 5.3 and the Cauchy-Schwarz inequality.
Recursively applying (B.5), we obtain for any  $i \geq 1$  that
$$
\left\| \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} \right\| _ {2} \leq L _ {\widehat {f}} L _ {\theta} \cdot \sum_ {j = 0} ^ {i - 1} L _ {\widehat {f}} ^ {j} \widetilde {L} _ {\pi} ^ {j} \leq i \cdot L _ {\theta} L _ {\widehat {f}} ^ {i + 1} \widetilde {L} _ {\pi} ^ {i}, \tag {B.6}
$$
where the first inequality follows from the induction
$$
z _ {i} = a z _ {i - 1} + b = a \cdot \left(a z _ {i - 2} + b\right) + b = a ^ {i} \cdot z _ {0} + b \cdot \sum_ {j = 0} ^ {i - 1} a ^ {j}. \tag {B.7}
$$
In (B.7),  $\{z_j\}_{0\leq j\leq i}$  is the real sequence satisfying  $z_{j} = az_{j - 1} + b$  . For  $\mathrm{d}\widehat{a}_{i,n} / \mathrm{d}\theta$  defined in (B.3), we further have
$$
\left\| \frac {\mathrm {d} \widehat {a} _ {i , n}}{\mathrm {d} \theta} \right\| _ {2} \leq L _ {\pi} \cdot \left\| \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} \right\| _ {2} + L _ {\theta} \leq i \cdot L _ {\theta} L _ {\widehat {f}} ^ {i + 1} \widetilde {L} _ {\pi} ^ {i + 1} + L _ {\theta}. \tag {B.8}
$$
Combining (B.6) and (B.8), we obtain
$$
\left\| \frac {\mathrm {d} \widehat {x} _ {i , n}}{\mathrm {d} \theta} \right\| _ {2} = \left\| \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} \right\| _ {2} + \left\| \frac {\mathrm {d} \widehat {a} _ {i , n}}{\mathrm {d} \theta} \right\| _ {2} \leq \underbrace {2 i \cdot L _ {\theta} L _ {\widehat {f}} ^ {i + 1} \widetilde {L} _ {\pi} ^ {i + 1} + L _ {\theta}} _ {\widehat {K} (i)}. \tag {B.9}
$$
Therefore, we bound the second term on the right-hand side of (B.2) as follows,
$$
\begin{array}{l} \mathbb {E} _ {\overline {{x}} _ {h}} \left[ \left\| \nabla \widehat {Q} (\widehat {x} _ {h, n}) \nabla_ {\theta} \widehat {x} _ {h, n} - \nabla \widehat {Q} (\overline {{x}} _ {h}) \nabla_ {\theta} \overline {{x}} _ {h} \right\| _ {2} \right] \\ \leq \mathbb {E} _ {\overline {{x}} _ {h}} \left[ \left\| \nabla \widehat {Q} (\widehat {x} _ {h, n}) \nabla_ {\theta} \widehat {x} _ {h, n} - \nabla \widehat {Q} (\overline {{x}} _ {h}) \nabla_ {\theta} \widehat {x} _ {h, n} \right\| _ {2} \right] + \mathbb {E} _ {\overline {{x}} _ {h}} \left[ \left\| \nabla \widehat {Q} (\overline {{x}} _ {h}) \nabla_ {\theta} \widehat {x} _ {h, n} - \nabla \widehat {Q} (\overline {{x}} _ {h}) \nabla_ {\theta} \overline {{x}} _ {h} \right\| _ {2} \right] \\ \leq 2 L _ {\widehat {Q}} \cdot \widehat {K} (i) + L _ {\widehat {Q}} \cdot \left(\mathbb {E} _ {\bar {s} _ {i}} \left[ \left\| \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {s} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right] + \mathbb {E} _ {\bar {a} _ {i}} \left[ \left\| \frac {\mathrm {d} \widehat {a} _ {i , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {a} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right]\right), \tag {B.10} \\ \end{array}
$$
where the last inequality follows from the Cauchy-Schwartz inequality and Assumption 5.3, and  $L_{\widehat{Q}} = \sup_{\omega \in \Omega ,s_1,s_2\in \mathcal{S},a_1,a_2\in \mathcal{A}}|\widehat{Q}_\omega (s_1,a_1) - \widehat{Q}_\omega (s_2,a_2)| / \| (s_1 - s_2,a_1 - a_2)\| _2$
By the chain rule, we bound the first term on the right-hand side of (B.2) as follows,
$$
\begin{array}{l} \mathbb {E} _ {\overline {{x}} _ {i}} \left[ \left\| \nabla_ {\theta} r (\widehat {x} _ {i, n}) - \nabla_ {\theta} r (\overline {{x}} _ {i}) \right\| _ {2} \right] \\ = \mathbb {E} _ {\overline {{x}} _ {i}} \left[ \left\| \nabla r (\widehat {x} _ {i, n}) \nabla_ {\theta} \widehat {x} _ {i, n} - \nabla r (\overline {{x}} _ {i}) \nabla_ {\theta} \overline {{x}} _ {i} \right\| _ {2} \right] \\ \leq \mathbb {E} _ {\overline {{x}} _ {i}} \left[ \left\| \nabla r (\widehat {x} _ {i, n}) \nabla_ {\theta} \widehat {x} _ {i, n} - \nabla r (\widehat {x} _ {i, n}) \nabla_ {\theta} \overline {{x}} _ {i} \right\| _ {2} \right] + \mathbb {E} \left[ \left\| \nabla r (\widehat {x} _ {i, n}) \nabla_ {\theta} \overline {{x}} _ {i} - \nabla r (\overline {{x}} _ {i}) \nabla_ {\theta} \overline {{x}} _ {i} \right\| _ {2} \right] \\ \leq L _ {r} \cdot \left(\mathbb {E} _ {\bar {s} _ {i}} \left[ \left\| \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {s} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right] + \mathbb {E} _ {\bar {a} _ {i}} \left[ \left\| \frac {\mathrm {d} \widehat {a} _ {i , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {a} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right]\right) + 2 L _ {r} \cdot \widehat {K} (i). \tag {B.11} \\ \end{array}
$$
Plugging (B.10) and (B.11) into (B.2) and (B.1), we obtain
$$
\begin{array}{l} v _ {t} \leq \left[ \left(L _ {r} \cdot \sum_ {i = 0} ^ {h - 1} \gamma^ {i} + \gamma^ {h} \cdot L _ {\widehat {Q}}\right) \cdot \left(\mathbb {E} _ {\bar {s} _ {h}} \left[ \left\| \frac {\mathrm {d} \widehat {s} _ {h , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {s} _ {h}}{\mathrm {d} \theta} \right\| _ {2} \right] + \mathbb {E} _ {\bar {a} _ {h}} \left[ \left\| \frac {\mathrm {d} \widehat {a} _ {h , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {a} _ {h}}{\mathrm {d} \theta} \right\| _ {2} \right] + 2 \widehat {K} (h)\right) \right] ^ {2} \\ = O \left(h ^ {4} \left(\frac {1 - \gamma^ {h}}{1 - \gamma}\right) ^ {2} \widetilde {L} _ {\widehat {f}} ^ {4 h} \widetilde {L} _ {\pi} ^ {4 h} + \gamma^ {2 h} h ^ {4} \widetilde {L} _ {\widehat {f}} ^ {4 h} \widetilde {L} _ {\pi} ^ {4 h}\right), \tag {B.12} \\ \end{array}
$$
where the inequality follows from Lemma B.2 and by plugging the definition of  $\widehat{K}(i)$  in (B.9).
Note that the variance  $v_{t}$  scales with the batch size  $N$  at the rate of  $1 / N$ . Since the analysis above is established for  $N = 1$ , the bound of the gradient variance  $v_{t}$  is established by dividing the right-hand side of (B.12) by  $N$ , which concludes the proof of Proposition 5.4.
Lemma B.2. Denote  $e = \sup \mathbb{E}_{\overline{s}_0}[\| \mathrm{d}\widehat{s}_{0,n} / \mathrm{d}\theta - \mathrm{d}\overline{s}_0 / \mathrm{d}\theta \|_2]$ , which is a constant that only depends on the initial state distribution<sup>2</sup>. For any timestep  $i \geq 1$  and the corresponding state, action, we have the following results,
$$
\mathbb {E} _ {\overline {{s}} _ {i}} \left[ \left\| \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \overline {{s}} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right] \leq \widetilde {L} _ {\widehat {f}} ^ {i} \widetilde {L} _ {\pi} ^ {i} \Big (e + 4 i \cdot \widetilde {L} _ {\widehat {f}} \widetilde {L} _ {\pi} \cdot \widehat {K} (i - 1) + 2 i \cdot \widetilde {L} _ {\widehat {f}} L _ {\theta} \Big),
$$
$$
\mathbb {E} _ {\bar {a} _ {i}} \left[ \left\| \frac {\mathrm {d} \widehat {a} _ {i , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {a} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right] \leq \widetilde {L} _ {\widehat {f}} ^ {i} \widetilde {L} _ {\pi} ^ {i + 1} \left(e + 4 i \cdot \widetilde {L} _ {\widehat {f}} \widetilde {L} _ {\pi} \cdot \widehat {K} (i - 1) + 2 i \cdot \widetilde {L} _ {\widehat {f}} L _ {\theta}\right) + 2 L _ {\pi} \widehat {K} (i) + 2 L _ {\theta}.
$$
Proof. Firstly, from (B.4), we obtain for any  $i \geq 1$  that
$$
\begin{array}{l} \mathbb {E} _ {\bar {s} _ {i}} \left[ \left\| \frac {\mathrm {d} \hat {s} _ {i , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {s} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right] \\ = \mathbb {E} \left[ \left\| \frac {\partial \widehat {s} _ {i , n}}{\partial \widehat {s} _ {i - 1 , n}} \cdot \frac {\mathrm {d} \widehat {s} _ {i - 1 , n}}{\mathrm {d} \theta} + \frac {\partial \widehat {s} _ {i , n}}{\partial \widehat {a} _ {i - 1 , n}} \cdot \frac {\mathrm {d} \widehat {a} _ {i - 1 , n}}{\mathrm {d} \theta} - \frac {\partial \overline {{s}} _ {i}}{\partial \overline {{s}} _ {i - 1}} \cdot \frac {\mathrm {d} \overline {{s}} _ {i - 1}}{\mathrm {d} \theta} - \frac {\partial \overline {{s}} _ {i}}{\partial \overline {{a}} _ {i - 1}} \cdot \frac {\mathrm {d} \overline {{a}} _ {i - 1}}{\mathrm {d} \theta} \right\| _ {2} \right] \\ \end{array}
$$
According to the triangle inequality, we further have
$$
\begin{array}{l} \leq \mathbb {E} \left[ \left\| \frac {\partial \widehat {s} _ {i , n}}{\partial \widehat {s} _ {i - 1 , n}} \cdot \frac {\mathrm {d} \widehat {s} _ {i - 1 , n}}{\mathrm {d} \theta} - \frac {\partial \overline {{s}} _ {i}}{\partial \overline {{s}} _ {i - 1}} \cdot \frac {\mathrm {d} \widehat {s} _ {i - 1 , n}}{\mathrm {d} \theta} \right\| _ {2} \right] + \mathbb {E} \left[ \left\| \frac {\partial \overline {{s}} _ {i}}{\partial \overline {{s}} _ {i - 1}} \cdot \frac {\mathrm {d} \widehat {s} _ {i - 1 , n}}{\mathrm {d} \theta} - \frac {\partial \overline {{s}} _ {i}}{\partial \overline {{s}} _ {i - 1}} \cdot \frac {\mathrm {d} \overline {{s}} _ {i - 1}}{\mathrm {d} \theta} \right\| _ {2} \right] \\ + \mathbb {E} \left[ \left\| \frac {\partial \widehat {s} _ {i , n}}{\partial \widehat {a} _ {i - 1 , n}} \cdot \frac {\mathrm {d} \widehat {a} _ {i - 1 , n}}{\mathrm {d} \theta} - \frac {\partial \overline {{s}} _ {i}}{\partial \overline {{a}} _ {i - 1}} \cdot \frac {\mathrm {d} \widehat {a} _ {i - 1 , n}}{\mathrm {d} \theta} \right\| _ {2} \right] + \mathbb {E} \left[ \left\| \frac {\partial \overline {{s}} _ {i}}{\partial \overline {{a}} _ {i - 1}} \cdot \frac {\mathrm {d} \widehat {a} _ {i - 1 , n}}{\mathrm {d} \theta} - \frac {\partial \overline {{s}} _ {i}}{\partial \overline {{a}} _ {i - 1}} \cdot \frac {\mathrm {d} \overline {{a}} _ {i - 1}}{\mathrm {d} \theta} \right\| _ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} \leq 2 L _ {\widehat {f}} \cdot \left(\left\| \frac {\mathrm {d} \widehat {s} _ {i - 1 , n}}{\mathrm {d} \theta} \right\| _ {2} + \left\| \frac {\mathrm {d} \widehat {a} _ {i - 1 , n}}{\mathrm {d} \theta} \right\| _ {2}\right) + L _ {\widehat {f}} \cdot \mathbb {E} _ {\bar {s} _ {i - 1}} \left[ \left\| \frac {\mathrm {d} \widehat {s} _ {i - 1 , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {s} _ {i - 1}}{\mathrm {d} \theta} \right\| _ {2} \right] \\ + L _ {\widehat {f}} \cdot \mathbb {E} _ {\bar {a} _ {i - 1}} \left[ \left\| \frac {\mathrm {d} \widehat {a} _ {i - 1 , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {a} _ {i - 1}}{\mathrm {d} \theta} \right\| _ {2} \right]. \tag {B.13} \\ \end{array}
$$
Similarly, we have from (B.3) that
$$
\begin{array}{l} \mathbb {E} _ {\bar {a} _ {i}} \left[ \left\| \frac {\mathrm {d} \hat {a} _ {i , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {a} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right] \\ = \mathbb {E} \left[ \left\| \frac {\partial \widehat {a} _ {i , n}}{\partial \widehat {s} _ {i , n}} \cdot \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} + \frac {\partial \widehat {a} _ {i , n}}{\partial \theta} - \frac {\partial \overline {{a}} _ {i}}{\partial \overline {{s}} _ {i}} \cdot \frac {\mathrm {d} \overline {{s}} _ {i}}{\mathrm {d} \theta} - \frac {\partial \overline {{a}} _ {i}}{\partial \theta} \right\| _ {2} \right] \\ \leq \mathbb {E} \left[ \left\| \frac {\partial \widehat {a} _ {i , n}}{\partial \widehat {s} _ {i , n}} \cdot \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} - \frac {\partial \overline {{a}} _ {i}}{\partial \overline {{s}} _ {i}} \cdot \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} \right\| _ {2} \right] + \mathbb {E} \left[ \left\| \frac {\partial \overline {{a}} _ {i}}{\partial \overline {{s}} _ {i}} \cdot \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} - \frac {\partial \overline {{a}} _ {i}}{\partial \overline {{s}} _ {i}} \cdot \frac {\mathrm {d} \overline {{s}} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right] + \mathbb {E} \left[ \left\| \frac {\partial \widehat {a} _ {i , n}}{\partial \theta} - \frac {\partial \overline {{a}} _ {i}}{\partial \theta} \right\| _ {2} \right] \\ \leq 2 L _ {\pi} \cdot \mathbb {E} \left[ \left\| \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} \right\| \right] + L _ {\pi} \cdot \mathbb {E} \left[ \left\| \frac {\mathrm {d} \widehat {s} _ {i , n}}{\mathrm {d} \theta} - \frac {\mathrm {d} \bar {s} _ {i}}{\mathrm {d} \theta} \right\| _ {2} \right] + 2 L _ {\theta}. \tag {B.14} \\ \end{array}
$$