SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\underset {a _ {1} \sim \pi} {\mathbb {E}} \left[ f _ {1} ^ {k} (s _ {1}, a _ {1}) \right] \stackrel {(i)} {\leq} \sqrt {\underset {a _ {1} \sim \pi} {\mathbb {E}} \left[ f _ {1} ^ {k} (s _ {1} , a _ {1}) ^ {2} \right]} \stackrel {(i i)} {\leq} \sqrt {A \left(\zeta_ {k , W} + 2 C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P}\right)},
$$
where  $(i)$  follows from Jensen's inequality, and  $(ii)$  follows from the importance sampling.
Then for  $h \geq 2$ , we derive the following bound:
$$
\begin{array}{l} \underset {(s _ {h}, a _ {h}) \sim (\hat {P} ^ {k}, \pi)} {\mathbb {E}} \left[ f _ {h} ^ {k} (s _ {h}, a _ {h}) \Bigg | s _ {h - 1}, a _ {h - 1} \right] \\ \stackrel {(i)} {\leq} \underset {a _ {h - 1} \sim \pi} {\mathbb {E}} \left[ \left\| \hat {\phi} _ {h - 1} ^ {k} (s _ {h - 1}, a _ {h - 1}) \right\| _ {(U _ {h - 1, \hat {\phi} ^ {k}} ^ {k}) ^ {- 1}} \times \left(A \sum_ {i = 1 \vee (k - W)} ^ {k - 1} \underset { \begin{array}{c} s _ {h - 1} \sim (P ^ {\star , i}, \pi^ {i}) \\ a _ {h - 1}, a _ {h} \sim \mathcal {U} (\mathcal {A}) \\ s _ {h} \sim P ^ {\star , i} (\cdot | s _ {h - 1}, a _ {h - 1}) \end{array} } {\mathbb {E}} [ f _ {h} ^ {k} (s _ {h}, a _ {h}) ^ {2} ] \right. \right. \\ \left. + W \Delta_ {h - 1, [ k - W, k - 1 ]} ^ {P} + \lambda_ {k, W} d + A \sum_ {i = 1 \vee (k - W)} ^ {k - 1} \underset { \begin{array}{c} s _ {h - 2} \sim (P ^ {\star , i}, \pi^ {i}) \\ a _ {h - 2}, a _ {h - 1} \sim \mathcal {U} (\mathcal {A}) \\ s _ {h - 1} \sim P ^ {\star , i} (\cdot | s _ {h - 2}, a _ {h - 2}) \end{array} } {\mathbb {E}} \left[ f _ {h - 1} ^ {k} (s _ {h - 1}, a _ {h - 1}) ^ {2} \right]\right) ^ {- \frac {1}{2}} \\ \stackrel {(i i)} {\leq} \mathbb {E} _ {a _ {h - 1} \sim \pi} \left[ \sqrt {W A (\zeta_ {k , W} + 2 C _ {B} \Delta_ {h , [ k - W , k - 1 ]} ^ {P}) + W A (\zeta_ {k , W} + 2 C _ {B} \Delta_ {h - 1 , [ k - W , k - 1 ]} ^ {P}) + W \Delta_ {h - 1 , [ k - W , k - 1 ]} ^ {P} + \lambda_ {k , W} d} \right. \\ \end{array}
$$
$$
\begin{array}{l} \times \left. \left\| \hat {\phi} _ {h - 1} ^ {k} \left(s _ {h - 1}, a _ {h - 1}\right) \right\| _ {\left(U _ {h - 1, \hat {\phi} ^ {k}} ^ {k}\right) ^ {- 1}} \right] \\ \stackrel {(i i i)} {=} \underset {a _ {h - 1} \sim \pi} {\mathbb {E}} \left[ \alpha_ {k, W} \left\| \hat {\phi} _ {h - 1} ^ {k} (s _ {h - 1}, a _ {h - 1}) \right\| _ {(U _ {h - 1, \hat {\phi} ^ {k}} ^ {k}) ^ {- 1}} \right] + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}}, \\ \end{array}
$$
where  $(i)$  follows from Lemma A.12 and because  $|f_h^k (s_h,a_h)|\leq 1$ ; specially the first term inside the square root follows from the definition of  $U_{h - 1,\widehat{\phi}^k}^k$ , the third term inside the square root follows from the importance sampling;  $(ii)$  follows from Lemma A.10, and  $(iii)$  follows because  $\sqrt{a + b}\leq \sqrt{a} +\sqrt{b},\forall a,b\geq 0$  and  $\left\| \hat{\phi}_{h - 1}^{k}(s_{h - 1},a_{h - 1})\right\|_{(U_{h - 1,\hat{\phi}^k}^k)^{-1}}\leq \sqrt{\frac{1}{\lambda_W}}.$
The next lemma follows a similar argument to that of Lemma A.13 with the only difference being the expectation over which  $f_h^k (s_h,a_h)$  takes.
Lemma A.14. Denote  $\alpha_{k,W} = \sqrt{2WA\zeta_{k,W} + \lambda_{k,W}d}$ . For any  $k \in [K]$ , policy  $\pi$  and reward  $r$ , for all  $h \geq 2$ , we have
$$
\begin{array}{l} \underset {(s _ {h}, a _ {h}) \sim (P ^ {\star , k}, \pi)} {\mathbb {E}} \left[ f _ {h} ^ {k} (s _ {h}, a _ {h}) \Bigg | s _ {h - 1}, a _ {h - 1} \right] \\ \leq \min  \left\{\alpha_ {k, W} \left\| \phi_ {h - 1} ^ {\star , k} \left(s _ {h - 1}, a _ {h - 1}\right) \right\| _ {\left(U _ {h - 1, \phi^ {\star}, k}\right) ^ {- 1}}, 1 \right\} + \sqrt {\frac {1}{2 d} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]}} \tag {24} \\ \end{array}
$$
$$
\underset {a _ {1} \sim \pi} {\mathbb {E}} \left[ f _ {1} ^ {k} \left(s _ {1}, a _ {1}\right) \right] \leq \sqrt {A \left(\zeta_ {k , W} + \frac {1}{2} C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P}\right)}. \tag {25}
$$
Proof. For  $h = 1$ ,
$$
\underset {a _ {1} \sim \pi} {\mathbb {E}} \left[ f _ {1} ^ {k} (s _ {1}, a _ {1}) \right] \stackrel {(i)} {\leq} \sqrt {\underset {a _ {1} \sim \pi} {\mathbb {E}} \left[ f _ {1} ^ {k} (s _ {1} , a _ {1}) ^ {2} \right]} \stackrel {(i i)} {\leq} \sqrt {A \left(\zeta_ {k , W} + \frac {1}{2} C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P}\right)},
$$
$$
\begin{array}{l} \underset {(s _ {h}, a _ {h}) \sim (P ^ {\star , k}, \pi)} {\mathbb {E}} \left[ f _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) \Bigg | s _ {h - 1}, a _ {h - 1} \right] \\ \overset {(i)} {\leq} \mathbb {E} _ {a _ {h - 1} \sim \pi} \left[ \begin{array}{c c c} & & \\ & \big \| \phi_ {h - 1} ^ {\star , k} (s _ {h - 1}, a _ {h - 1}) \big \| _ {(U _ {h - 1, \phi^ {\star}, k}) ^ {- 1}} \times & A \sum_ {i = 1 \vee (k - W)} ^ {k - 1} \mathbb {E} _ { \begin{array}{c c c} & s _ {h - 1} \sim (P ^ {\star , i, \pi^ {i}}) & [ f _ {h} ^ {k} (s _ {h}, a _ {h}) ^ {2} ] + \lambda_ {k, W} d \\ & a _ {h - 1}, a _ {h} \sim \mathcal {U} (A) \\ & s _ {h} \sim P ^ {\star , k} (\cdot | s _ {h - 1}, a _ {h - 1}) \end{array} } & \end{array} \right] \\ \stackrel {(i i)} {\leq} \mathbb {E} _ {a _ {h - 1} \sim \pi} \left[ \sqrt {w A (\zeta_ {k , W} + \frac {1}{2} C _ {B} \Delta_ {h , [ k - W , k - 1 ]} ^ {P}) + w A (\zeta_ {k , W} + \frac {1}{2} C _ {B} \Delta_ {h - 1 , [ k - W , k - 1 ]} ^ {P}) + A C _ {B} \Delta_ {h , [ k - W , k - 1 ]} ^ {P} + \lambda_ {k , W} d} \right. \\ \left. \times \left\| \phi_ {h - 1} ^ {\star , k} \left(s _ {h - 1}, a _ {h - 1}\right) \right\| _ {\left(U _ {h - 1, \phi^ {\star}, k} ^ {k}\right) ^ {- 1}} \right] \\ \stackrel {(i i i)} {=} \underset {a _ {h - 1} \sim \pi} {\mathbb {E}} \left[ \alpha_ {k, W} \left\| \phi_ {h - 1} ^ {\star , k} (s _ {h - 1}, a _ {h - 1}) \right\| _ {(U _ {h - 1, \phi^ {\star}, k} ^ {k}) ^ {- 1}} \right] + \sqrt {\frac {1}{2 d} W A C _ {B} \Delta_ {[ h - 1 , h ], [ k - W , k - 1 ]} ^ {P}}, \\ \end{array}
$$
where  $(i)$  follows from Lemma A.12 and because  $|f_h^k (s_h,a_h)|\leq 1$ , the first term inside the square root follows from the definition of  $U_{h - 1,\widehat{\phi}^k}^k$ , the third term inside the square root follows from the importance sampling, and  $(ii)$  follows from Lemma A.10.
The proof is completed by noting that  $|f_h^k (s_h,a_h)|\leq 1$
The following lemma is a direct application of Lemma A.13. By this lemma, we can show that the difference of value functions can be bounded by truncated value function plus a variation term.
Lemma A.15 (Bounded difference of value functions). For  $k \in [K]$ ,  $\delta \geq 0$  any policy  $\pi$  and reward  $r$ , with probability at least  $1 - \delta$ , we have
$$
\begin{array}{l} \left| V _ {P ^ {\star}, k, r} ^ {\pi} - V _ {\hat {P} ^ {k}, r} ^ {\pi} \right| \\ \leq \hat {V} _ {\hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} + \sum_ {h = 2} ^ {H} \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}} + \sqrt {A \left(\zeta_ {k , W} + 2 C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P}\right)}. \tag {26} \\ \end{array}
$$
Proof. (1): We first show that  $\left|V_{P^{\star ,k},r}^{\pi} - V_{\hat{P}^{k},r}^{\pi}\right|\leq \hat{V}_{\hat{P}^{k},f^{k}}^{\pi}$ .
Recall the definition of the estimated value functions  $\hat{V}_{h,\hat{P}^k,r}^\pi (s_h)$  and  $\hat{Q}_{h,\hat{P}^{k},r}^{\pi}(s_h,a_h)$  for policy  $\pi$ :
$$
\begin{array}{l} \hat {Q} _ {h, \hat {P} ^ {k}, r} ^ {\pi} (s _ {h}, a _ {h}) = \min  \left\{1, r _ {h} (s _ {h}, a _ {h}) + \hat {P} _ {h} ^ {k} \hat {V} _ {h + 1, \hat {P} ^ {k}, r} ^ {\pi} (s _ {h}, a _ {h}) \right\}, \\ \hat {V} _ {h, \hat {P} ^ {k}, r} ^ {\pi} (s _ {h}) = \underset {\pi} {\mathbb {E}} \left[ \hat {Q} _ {h, \hat {P} ^ {k}, r} ^ {\pi} (s _ {h}, a _ {h}) \right]. \\ \end{array}
$$
We develop the proof by backward induction.
When  $h = H + 1$ , we have  $\left|V_{H + 1,P^{\star ,k},r}^{\pi}(s_{H + 1}) - V_{H + 1,\hat{P}^{k},r}^{\pi}(s_{H + 1})\right| = 0 = \hat{V}_{H + 1,\hat{P}^{k},f^{k}}^{\pi}(s_{H + 1}).$
Suppose that for  $h + 1$ ,  $\left|V_{h + 1,P^{\star ,k},r}^{\pi}(s_{h + 1}) - V_{h + 1,\hat{P}^{k},r}^{\pi}(s_{h + 1})\right|\leq \hat{V}_{h + 1,\hat{P}^{k},f^{k}}^{\pi}(s_{h + 1})$  holds for any  $s_{h + 1}$ .
Then, for  $h$ , by Bellman equation, we have,
$$
\begin{array}{l} \left| Q _ {P ^ {\star , k}, r} ^ {\pi} \left(s _ {h}, a _ {h}\right) - Q _ {h, \hat {P} _ {k}, r} ^ {\pi} \left(s _ {h}, a _ {h}\right) \right| \\ = \left| P _ {h} ^ {\star , k} V _ {h + 1, P ^ {\star , k}, r} ^ {\pi} (s _ {h}, a _ {h}) - \hat {P} _ {h} ^ {k} V _ {h, \hat {P} ^ {k}, r} ^ {\pi} (s _ {h}, a _ {h}) \right| \\ = \left| \hat {P} _ {h} ^ {k} \left(V _ {h + 1, P ^ {\star , k}, r} ^ {\pi} - V _ {h + 1, \hat {P} ^ {k}, r} ^ {\pi}\right) (s _ {h}, a _ {h}) + \left(P _ {h} ^ {\star , k} - \hat {P} _ {h} ^ {k}\right) V _ {h, P ^ {\star , k}, r} ^ {\pi} (s _ {h}, a _ {h}) \right| \\ \stackrel {(i)} {\leq} \min  \left\{1, f _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) + \hat {P} _ {h} ^ {k} \left| V _ {h + 1, P ^ {\star , k}, r} ^ {\pi} - V _ {h + 1, \hat {P} ^ {k}, r} ^ {\pi} \right| \left(s _ {h}, a _ {h}\right) \right\} \\ \stackrel {(i i)} {\leq} \min  \left\{1, f _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) + \hat {P} _ {h} ^ {k} \hat {V} _ {h + 1, \hat {P} ^ {k}, f ^ {k}} ^ {\pi} \left(s _ {h}, a _ {h}\right) \right\} \\ = \hat {Q} _ {h, \hat {P} ^ {k}, f ^ {k}} ^ {\pi} \left(s _ {h}, a _ {h}\right), \tag {27} \\ \end{array}
$$
where  $(i)$  follows because  $\left\| \hat{P}_h^k (\cdot |s_h,a_h) - P_h^{\star ,k}(\cdot |s_h,a_h)\right\|_{TV} = f_h^k (s_h,a_h)$  and the value function is at most 1, and  $(ii)$  follows from the induction hypothesis.
Then, by the definition of  $\hat{V}_{h,\hat{P}^k,r}^\pi (s_h)$ , we have
$$
\begin{array}{l} \left| V _ {h, \hat {P} ^ {k}, r} ^ {\pi} (s _ {h}) - V _ {h, P ^ {\star , k}, r} ^ {\pi} (s _ {h}) \right| \\ = \left| \mathbb {E} _ {\pi} \left[ Q _ {h, \hat {P} ^ {k}, r} ^ {\pi} (s _ {h}, a _ {h}) \right] - \mathbb {E} _ {\pi} \left[ Q _ {h, P ^ {\star , k}, r} ^ {\pi} (s _ {h}, a _ {h}) \right] \right| \\ \leq \mathbb {E} _ {\pi} \left[ \left| Q _ {h, \hat {P} ^ {k}, r} ^ {\pi} \left(s _ {h}, a _ {h}\right) - Q _ {h, P ^ {\star , k}, r} ^ {\pi} \left(s _ {h}, a _ {h}\right) \right| \right] \\ \stackrel {(i)} {\leq} \underset {\pi} {\mathbb {E}} \left[ \hat {Q} _ {h, \hat {P} ^ {k}, f ^ {k}} ^ {\pi} \left(s _ {h}, a _ {h}\right) \right] \\ \end{array}
$$
$$
= \hat {V} _ {h, \hat {P} ^ {k}, f ^ {k}} ^ {\pi} (s _ {h}),
$$
where  $(i)$  follows from Equation (27).
Therefore, by induction, we have
$$
\left| V _ {P ^ {\star , k}, r} ^ {\pi} - V _ {\hat {P} ^ {k}, r} ^ {\pi} \right| \leq \hat {V} _ {\hat {P} ^ {k}, f ^ {k}} ^ {\pi}.
$$
(2): Then we prove that
$$
\hat {V} _ {\hat {P} ^ {k}, f ^ {k}} ^ {\pi} \leq \hat {V} _ {\hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} + \sum_ {h = 2} ^ {H} \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}} + \sqrt {A (\zeta_ {k , W} + 2 C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P})}.
$$
By Lemma A.13 and the fact that the total variation distance is upper bounded by  $1, \forall h \geq 2$ , with probability at least  $1 - \delta / 2$ , we have
$$
\begin{array}{l} \underset {\hat {P} ^ {k}, \pi} {\mathbb {E}} \left[ f _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) \Big | s _ {h - 1} \right] \leq \underset {\pi} {\mathbb {E}} \left[ \min  \left(\alpha_ {k, W} \left\| \hat {\phi} _ {h - 1} ^ {k} \left(s _ {h - 1}, a _ {h - 1}\right) \right\| _ {\left(U _ {h - 1, \hat {\phi} ^ {k}} ^ {k}\right) ^ {- 1}}, 1\right) \right] \\ + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}}. \tag {28} \\ \end{array}
$$
Similarly, when  $h = 1$
$$
\underset {a _ {1} \sim \pi} {\mathbb {E}} \left[ f _ {1} ^ {k} \left(s _ {1}, a _ {1}\right) \right] \leq \sqrt {A \left(\zeta_ {k , W} + 2 C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P}\right)}. \tag {29}
$$
Based on Corollary A.9, Equation (28) and  $\tilde{\alpha}_{k,W} = 5\alpha_{k,W}$ , we have
$$
\begin{array}{l} \underset {\pi} {\mathbb {E}} \left[ \hat {b} _ {h} ^ {k} (s _ {h}, a _ {h}) \Big | s _ {h} \right] + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h , h + 1 \} , [ k - W , k - 1 ]} ^ {P}} \\ \geq \mathbb {E} _ {\pi} \left[ \min  \left(\alpha_ {k, W} \left\| \hat {\phi} _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) \right\| _ {\left(U _ {h, \hat {\phi} ^ {k}} ^ {k}\right) ^ {- 1}}, 1\right) \right] + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}} \\ \geq \mathbb {E} _ {\hat {P} ^ {k}, \pi} \left[ f _ {h + 1} ^ {k} \left(s _ {h + 1}, a _ {h + 1}\right) \mid s _ {h} \right]. \tag {30} \\ \end{array}
$$
For the base case  $h = H$ , we have
$$
\begin{array}{l} \underset {\hat {P} ^ {k}, \pi} {\mathbb {E}} \left[ \hat {V} _ {H, \hat {P} ^ {k}, f ^ {k}} ^ {\pi} (s _ {H}) \Bigg | s _ {h - 1}, a _ {h - 1} \right] \\ = \underset {\hat {P} ^ {k}, \pi} {\mathbb {E}} \left[ f _ {H} ^ {k} \left(s _ {H}, a _ {H}\right) \Bigg | s _ {H - 1} \right] \\ \leq \underset {\pi} {\mathbb {E}} \left[ b _ {H - 1} ^ {k} \left(s _ {H - 1}, a _ {H - 1}\right) \mid s _ {H - 1} \right] + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{H - 1 , H \} , [ k - W , k - 1 ]} ^ {P}} \\ \leq \min \left\{1, _ {\mathbb {E} _ {\pi}} \left[ \hat {Q} _ {H - 1, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} (s _ {H - 1}, a _ {H - 1}) \Bigg | s _ {h - 1}, a _ {h - 1} \right] \right\} + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{H - 1 , H \} , [ k - W , k - 1 ]} ^ {P}} \\ = \hat {V} _ {H - 1, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} (s _ {H - 1}) + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{H - 1 , H \} , [ k - W , k - 1 ]} ^ {P}}. \\ \end{array}
$$
For any step  $h + 1, h \geq 2$ , assume that  $\mathbb{E}_{\hat{P}^{k}, \pi}\left[\hat{V}_{h + 1, \hat{P}^{k}, f^{k}}^{\pi}(s_{h + 1}) \Big| s_{h}\right] \leq \hat{V}_{h, \hat{P}^{k}, \hat{b}^{k}}^{\pi}(s_{h}) + \sum_{h' = h + 1}^{H} \sqrt{\frac{3}{\lambda_{W}} W A C_{B} \Delta_{\{h' - 1, h'\}, [k - W, k - 1]}^{P}}$  holds. Then, by Jensen's inequality, we obtain
$$
\underset {\hat {P} ^ {k}, \pi} {\mathbb {E}} \left[ \hat {V} _ {h, \hat {P} ^ {k}, f ^ {k}} ^ {\pi} (s _ {h}) \Bigg | s _ {h - 1}, a _ {h - 1} \right]
$$
$$
\begin{array}{l} \leq \min  \left\{1, \underset {\hat {P} ^ {k}, \pi} {\mathbb {E}} \left[ f _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) + \hat {P} _ {h} ^ {k} \hat {V} _ {h + 1, \hat {P} ^ {k}, f ^ {k}} ^ {\pi} \left(s _ {h}, a _ {h}\right) \Bigg | s _ {h - 1}, a _ {h - 1} \right] \right\} \\ \stackrel {(i)} {\leq} \min  \left\{1, \underset {\pi} {\mathbb {E}} \left[ \hat {b} _ {h - 1} ^ {k} \left(s _ {h - 1}, a _ {h - 1}\right) \right] + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}} \right. \\ \left. + \underset {\hat {P} ^ {k}, \pi} {\mathbb {E}} \left[ \underset {\hat {P} ^ {k}, \pi} {\mathbb {E}} \left[ \hat {V} _ {h + 1, \hat {P} ^ {k}, f ^ {k}} ^ {\pi} (s _ {h + 1}) \Big | s _ {h} \right] \Big | s _ {h - 1}, a _ {h - 1} \right] \right\} \\ \stackrel {(i i)} {\leq} \min  \left\{1, \underset {\pi} {\mathbb {E}} \left[ b _ {h - 1} ^ {k} \left(s _ {h - 1}, a _ {h - 1}\right) \right] + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}} \right. \\ + \underset {\hat {P} ^ {k}, \pi} {\mathbb {E}} \left[ \hat {V} _ {h, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} (s _ {h}) \Bigg | s _ {h - 1}, a _ {h - 1} \right] + \sum_ {h ^ {\prime} = h + 1} ^ {H} \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h ^ {\prime} - 1 , h ^ {\prime} \} , [ k - W , k - 1 ]} ^ {P}} \Bigg \} \\ = \min  \left\{1, \mathbb {E} _ {\pi} \left[ \hat {Q} _ {h - 1, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} \left(s _ {h - 1}, a _ {h - 1}\right) \right] \right\} + \sum_ {h ^ {\prime} = h} ^ {H} \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h ^ {\prime} - 1 , h ^ {\prime} \} , [ k - W , k - 1 ]} ^ {P}} \\ = \hat {V} _ {h - 1, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} (s _ {h - 1}) + \sum_ {h ^ {\prime} = h} ^ {H} \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h ^ {\prime} - 1 , h ^ {\prime} \} , [ k - W , k - 1 ]} ^ {P}}, \\ \end{array}
$$
where  $(i)$  follows from Equation (30), and  $(ii)$  is due to the induction hypothesis.
By induction, we conclude that
$$
\begin{array}{l} \hat {V} _ {\hat {P} ^ {k}, f ^ {k}} ^ {\pi} = \underset {\pi} {\mathbb {E}} \left[ f _ {1} ^ {(s)} \left(s _ {1}, a _ {1}\right) \right] + \underset {\hat {P} ^ {k}, \pi} {\mathbb {E}} \left[ \hat {V} _ {2, \hat {P} ^ {k}, f ^ {k}} ^ {\pi} \left(s _ {2}\right) \Big | s _ {1} \right] \\ \leq \sqrt {A \left(\zeta_ {k , W} + 2 C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P}\right)} + \hat {V} _ {\hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} + \sum_ {h ^ {\prime} = 2} ^ {H} \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h ^ {\prime} - 1 , h ^ {\prime} \} , [ k - W , k - 1 ]} ^ {P}}. \\ \end{array}
$$
Combining Step 1 and Step 2, we conclude that
$$
\begin{array}{l} \left| V _ {P ^ {*}, r} ^ {\pi} - V _ {\hat {P} ^ {k}, r} ^ {\pi} \right| \\ \leq \hat {V} _ {\hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} + \sqrt {A (\zeta_ {k , W} + 2 C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P})} + \sum_ {h ^ {\prime} = 2} ^ {H} \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h ^ {\prime} - 1 , h ^ {\prime} \} , [ k - W , k - 1 ]} ^ {P}}. \\ \end{array}
$$
Similarly to Lemma A.15, we can prove that the total variance distance is bounded by  $\hat{V}_{\hat{P}^k,\hat{b}^k}$  plus a variation budget term as follows. Lemmas A.15 and A.16 together justify the choice of exploration policy for the off-policy exploration.
Lemma A.16. Fix  $\delta \in (0,1)$ , for any  $h\in [H],k\in [K]$ , any policy  $\pi$ , with probability at least  $1 - \delta /2$
$$
\begin{array}{l} \underset { \begin{array}{c} s _ {h} \sim (P ^ {\star}, k, \pi) \\ s _ {h} \sim \pi \end{array} } {\mathbb {E}} \left[ f _ {h} ^ {k} (s _ {h}, a _ {h}) \right] \\ \leq 2 \left(\hat {V} _ {\hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} + \sum_ {h = 2} ^ {H} \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}} + \sqrt {A (\zeta_ {k , W} + 2 C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P})}\right). \\ \end{array}
$$
Proof. Fix any policy  $\pi$ , for any  $h \geq 2$ , we have
$$
\begin{array}{l} \underset { \begin{array}{c} s _ {h} \sim (\hat {P} ^ {k}, \pi) \\ a _ {h} \sim \pi \end{array} } {\mathbb {E}} \left[ \hat {Q} _ {h, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} (s _ {h}, a _ {h}) \right] \\ = \underset { \begin{array}{c} s _ {h - 1} \sim (\hat {P} ^ {k}, \pi) \\ a _ {h - 1} \sim \pi \end{array} } {\mathbb {E}} \left[ \hat {P} _ {h} ^ {k} \hat {V} _ {h, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} (s _ {h - 1}, a _ {h - 1}) \right] \\ \end{array}
$$
$$
\begin{array}{l} \leq \underset { \begin{array}{c} s _ {h - 1} \sim (\hat {P} ^ {k}, \pi) \\ a _ {h - 1} \sim \pi \end{array} } {\mathbb {E}} \left[ \min  \left\{1, \hat {b} _ {h - 1} ^ {k} (s _ {h - 1}, a _ {h - 1}) + \hat {P} _ {h - 1} ^ {k} \hat {V} _ {h, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} (s _ {h - 1}, a _ {h - 1}) \right\} \right] \\ = \underset { \begin{array}{c} s _ {h - 1} \sim (\hat {P} ^ {k}, \pi) \\ a _ {h - 1} \sim \pi \end{array} } {\mathbb {E}} \left[ \hat {Q} _ {h - 1, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} (s _ {h - 1}, a _ {h - 1}) \right] \\ \leq \dots \\ \leq \underset {a _ {1} \sim \pi} {\mathbb {E}} \left[ \hat {Q} _ {1, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} \left(s _ {1}, a _ {1}\right) \right] \\ = \hat {V} _ {\hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi}. \tag {31} \\ \end{array}
$$
Hence, for  $h \geq 2$ , we have
$$
\begin{array}{l} \underset { \begin{array}{c} s _ {h} \sim (\hat {P} ^ {k}, \pi) \\ a _ {h} \sim \pi \end{array} } {\mathbb {E}} \left[ f _ {h} ^ {k} (s _ {h}, a _ {h}) \right] \overset {(i)} {\leq} \underset { \begin{array}{c} s _ {h - 1} \sim (\hat {P} ^ {k}, \pi) \\ a _ {h - 1} \sim \pi \end{array} } {\mathbb {E}} \left[ \hat {b} _ {h - 1} ^ {k} (s _ {h - 1}, a _ {h - 1}) \right] + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}} \\ \stackrel {(i i)} {\leq} \underset { \begin{array}{c} s _ {h - 1} \sim (\hat {P} ^ {k}, \pi) \\ a _ {h - 1} \sim \pi \end{array} } {\mathbb {E}} \left[ \hat {Q} _ {h - 1, \hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} (s _ {h - 1}, a _ {h - 1}) \right] + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}} \\ \stackrel {(i i i)} {\leq} \hat {V} _ {\hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} + \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}}, \tag {32} \\ \end{array}
$$
where  $(i)$  follows from Equation (30),  $(ii)$  follows from the definition of  $\hat{Q}_{h - 1,\hat{P}^k,\hat{b}^k}^\pi (s_{h - 1},a_{h - 1})$ , and  $(iii)$  follows from Equation (31).
$$
\begin{array}{l} \underset { \begin{array}{c} s _ {h} \sim (P ^ {\star}, k, \pi) \\ s _ {h} \sim \pi \end{array} } {\mathbb {E}} \left[ f _ {h} ^ {k} (s _ {h}, a _ {h}) \right] \\ \leq \underset { \begin{array}{c} s _ {h} \sim (\hat {P} ^ {k}, \pi) \\ a _ {h} \sim \pi \end{array} } {\mathbb {E}} \left[ f _ {h} ^ {k} (s _ {h}, a _ {h}) \right] + \left| \underset { \begin{array}{c} s _ {h} \sim (P ^ {\star}, k, \pi) \\ a _ {h} \sim \pi \end{array} } {\mathbb {E}} \left[ f _ {h} ^ {k} (s _ {h}, a _ {h}) \right] - \underset { \begin{array}{c} s _ {h} \sim (\hat {P} ^ {k}, \pi) \\ a _ {h} \sim \pi \end{array} } {\mathbb {E}} \left[ f _ {h} ^ {k} (s _ {h}, a _ {h}) \right] \right| \\ \leq 2 \left(\hat {V} _ {\hat {P} ^ {k}, \hat {b} ^ {k}} ^ {\pi} + \sum_ {h = 2} ^ {H} \sqrt {\frac {3}{\lambda_ {W}} W A C _ {B} \Delta_ {\{h - 1 , h \} , [ k - W , k - 1 ]} ^ {P}} + \sqrt {A (\zeta_ {k , W} + 2 C _ {B} \Delta_ {1 , [ k - W , k - 1 ]} ^ {P})}\right), \\ \end{array}
$$
where the last equation follows from Equation (32) and Lemma A.15.
Lemma A.17. Denote  $\tilde{\alpha}_{k,W} = 5\alpha_{k,W}$ ,  $\alpha_{k,W} = \sqrt{2WA\zeta_{k,W} + \lambda_{k,W}d}$ , and  $\beta_{k,W} = \sqrt{9dA\alpha_{k,W}^2 + \lambda_{k,W}d}$ . For any  $k \in [K]$ , policy  $\pi$  and reward  $r$ , for all  $h \geq 2$ , we have
$$
\begin{array}{l} \underset {(s _ {h}, a _ {h}) \sim (P ^ {\star , k}, \pi)} {\mathbb {E}} \left[ \hat {b} _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) \Bigg | s _ {h - 1}, a _ {h - 1} \right] \\ \leq \beta_ {k, W} \left\| \phi_ {h - 1} ^ {\star , k} \left(s _ {h - 1}, a _ {h - 1}\right) \right\| _ {\left(W _ {h - 1, \phi^ {\star}, k} ^ {k}\right) ^ {- 1}} + \sqrt {\frac {A}{d}} \Delta_ {h - 1, [ k - W, k - 1 ]} ^ {\sqrt {P}}, \tag {33} \\ \end{array}
$$
$$
\underset {a _ {1} \sim \pi} {\mathbb {E}} \left[ \hat {b} _ {1} ^ {k} \left(s _ {1}, a _ {1}\right) \right] \leq \sqrt {\frac {9 A d \alpha_ {k , W} ^ {2}}{W}}. \tag {34}
$$
$$
\operatorname * {E} _ {a _ {1} \sim \pi} \left[ \hat {b} _ {1} ^ {k} (s _ {1}, a _ {1}) \right] \stackrel {(i)} {\leq} \sqrt {\operatorname * {E} _ {a _ {1} \sim \pi} \left[ \hat {b} _ {1} ^ {k} (s _ {1} , a _ {1}) ^ {2} \right]} \stackrel {(i i)} {\leq} \sqrt {\frac {9 A d \alpha_ {k , W} ^ {2}}{W}},
$$
Then for  $h \geq 2$ , we first notice that
$$
\begin{array}{l} \sum_{i = 1\vee (k - W)}^{k - 1}\underset { \begin{array}{c}s_{h}\sim \bigl(P^{\star ,i},\pi^{i}\bigr)\\ a_{h}\sim \mathcal{U}(\mathcal{A}) \end{array} }{\mathbb{E}}\left[\hat{b}_{h}^{k}(s_{h},a_{h})^{2}\right] \\ = \sum_{i = 1\vee (k - W)}^{k - 1}\mathbb{E}_{\substack{s_{h}\sim \left(P^{\star},i,\pi^{i}\right)\\ a_{h}\sim \mathcal{U}(\mathcal{A})}}\left[\alpha_{k,W}^{2}\left\| \hat{\phi}_{h}^{k}(s_{h},a_{h})\right\|_{(\hat{U}_{h}^{k})^{-1}}^{2}\right] \\ \leq \sum_{i = 1\vee (k - W)}^{k - 1}\underset { \begin{array}{c}s_{h}\sim (P^{\star ,i},\pi^{i})\\ a_{h}\sim \mathcal{U}(\mathcal{A}) \end{array} }{\mathbb{E}}\left[9\alpha_{k,W}^{2}\left\| \hat{\phi}_{h}^{k}(s_{h},a_{h})\right\|_{(U_{h,\hat{\phi}^{k}}^{k})^{-1}}^{2}\right] \\ \leq 9\alpha_{k,W}^{2}\mathrm{tr}\left(\sum_{i = 1\vee (k - W)}^{k - 1}\underset { \begin{array}{s_{h}\sim (P^{\star ,i},\pi^{i})\\ a_{h}\sim \mathcal{U}(\mathcal{A}) \end{array} }{\mathbb{E}}\left[\hat{\phi}_{h}^{k}(s_{h},a_{h})\hat{\phi}_{h}^{k}(s_{h},a_{h})^{\top}\right](U_{h,\hat{\phi}^{k}}^{k})^{-1}\right) \\ \leq 9 \alpha_ {k, W} ^ {2} \operatorname {t r} \left(I _ {d}\right) = 9 d \alpha_ {k, W} ^ {2}, \tag {35} \\ \end{array}
$$