SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
P (s ^ {\prime} | s, a) \geq \widehat {P} _ {t} (s ^ {\prime} | s, a) = \widetilde {P} _ {i (t)} (s ^ {\prime} | s, a), \forall (s, a, s ^ {\prime}) \in W _ {k}, k = 0, \dots , L - 1.
$$
Therefore, we can show that
$$
0 \leq \sum_ {s ^ {\prime} \in S _ {k (s) + 1}} \left(P \left(s ^ {\prime} | s, a\right) - \widehat {P} _ {t} \left(s ^ {\prime} | s, a\right)\right) V ^ {\widehat {P} _ {t}, \pi^ {\star}} \left(s ^ {\prime}; \alpha_ {t}\right)
$$
$$
= \mathcal {O} \left(L \left(\sqrt {\frac {| S | \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}} + \frac {| S | \left(C ^ {\mathsf {P}} + \log (\iota)\right)}{\widehat {m} _ {i (t)} (s , a)}\right)\right),
$$
where the last step follows from the definition of optimistic transition and the fact that  $\alpha_{t}(s,a)\leq 1$
By direct calculation, we have  $\mathrm{TERM}2(\pi^{\star})$  bounded by
$$
\begin{array}{l} \mathcal {O} \left(L \sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {t = 1} ^ {T} \left[ q ^ {P, \pi_ {t}} (s, a) - q ^ {P, \pi^ {\star}} (s, a) \right] _ {+} \left(\sqrt {\frac {| S | \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}} + \frac {| S | \left(C ^ {\mathsf {P}} + \log (\iota)\right)}{\widehat {m} _ {i (t)} (s , a)}\right)\right) \\ \leq \mathcal {O} \left(L \sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {t = 1} ^ {T} \left[ q ^ {P, \pi_ {t}} (s, a) - q ^ {P, \pi^ {\star}} (s, a) \right] _ {+} \sqrt {\frac {| S | \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}}\right) \\ + \mathcal {O} \left(L | S | \sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {t = 1} ^ {T} q ^ {P, \pi_ {t}} (s, a) \left(\frac {\left(C ^ {\mathsf {P}} + \log (\iota)\right)}{\widehat {m} _ {i (t)} (s , a)}\right)\right) \\ = \mathcal {O} \left(L \sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {t = 1} ^ {T} \left[ q ^ {P, \pi_ {t}} (s, a) - q ^ {P, \pi^ {\star}} (s, a) \right] _ {+} \sqrt {\frac {| S | \log (\iota)}{\hat {m} _ {i (t)} (s , a)}} + L | S | ^ {2} | A | \left(C ^ {\mathsf {P}} + \log (\iota)\right) \log (T)\right), \\ \end{array}
$$
where the last step follows from Lemma D.5.3.
$$
\begin{array}{l} \sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {t = 1} ^ {T} \left[ q ^ {P, \pi_ {t}} (s, a) - q ^ {P, \pi^ {\star}} (s, a) \right] _ {+} \sqrt {\frac {| S | \log (t)}{\widehat {m} _ {i (t)} (s , a)}} \\ \leq \sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {t = 1} ^ {T} \sqrt {\left[ q ^ {P , \pi_ {t}} (s , a) - q ^ {P , \pi^ {\star}} (s , a) \right] _ {+}} \cdot \sqrt {\frac {q ^ {P , \pi_ {t}} (s , a) | S | \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}} \\ \leq \sqrt {\sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {t = 1} ^ {T} \left[ q ^ {P , \pi_ {t}} (s , a) - q ^ {P , \pi^ {\star}} (s , a) \right] _ {+}} \cdot \sqrt {\sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {t = 1} ^ {T} \frac {q ^ {P , \pi_ {t}} (s , a) | S | \log (\iota)}{\hat {m} _ {i (t)} (s , a)}} \\ = \mathcal {O} \left(\sqrt {| S | ^ {2} | A | \log^ {2} (\iota) \sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {t = 1} ^ {T} [ q ^ {P , \pi_ {t}} (s , a) - q ^ {P , \pi^ {\star}} (s , a) ] _ {+}}\right) \\ \leq \mathcal {O} \left(\sqrt {L | S | ^ {2} | A | \log^ {2} (\iota) \sum_ {t = 1} ^ {T} \sum_ {s \neq s _ {L}} \sum_ {a \neq \pi^ {\star} (s)} q ^ {P , \pi_ {t}} (s , a)}\right) \\ \leq \mathcal {O} \left(\sqrt {L | S | ^ {2} | A | \log^ {2} (\iota) \sum_ {t = 1} ^ {T} \sum_ {s \neq s _ {L}} \sum_ {a \neq \pi^ {\star} (s)} q ^ {P _ {t} , \pi_ {t}} (s , a)} + \sqrt {L ^ {2} | S | ^ {2} | A | \log^ {2} C ^ {\mathbb {P}}}\right), \\ \end{array}
$$
where the second step uses the Cauchy-Schwarz inequality; the third step applies Lemma D.5.3; the fifth step follows from Corollary D.3.5; the last step uses Corollary D.3.6 and the fact that  $\sqrt{x + y} \leq \sqrt{x} + \sqrt{y}$  for any  $x, y \geq 0$ .
Finally, applying Lemma D.1.1 finishes the proof.
C.6.3 Bounding Term 3
Lemma C.6.4. For any  $\delta \in (0,1)$  and the policy  $\pi^{\star}$ , Algorithm 4 ensures that
$$
\mathbb {E} \left[ \mathrm {T e r m} 3 (\pi^ {\star}) \right] = \mathcal {O} \left(\mathbb {S} _ {1} \left(L ^ {3} | S | ^ {2} | A | \log^ {2} (\iota)\right) + \delta L | S | ^ {2} | A | T + \left(C ^ {\mathcal {P}} + \log (\iota)\right) L ^ {2} | S | ^ {4} | A | \log (\iota)\right).
$$
Proof. Suppose that  $\mathcal{E}_{\mathrm{CON}} \wedge \mathcal{E}_{\mathrm{EST}}$  occurs. We first have:
$$
\left\langle q ^ {P, \pi_ {t}}, \mathcal {Z} _ {\alpha_ {t}} ^ {\widehat {P} _ {t}, P, \pi^ {*}} - \mathcal {Z} _ {\alpha_ {t}} ^ {\widehat {P} _ {t}, P, \pi_ {t}} \right\rangle
$$
$$
\begin{array}{l} = \sum_ {s \neq s _ {L}} \sum_ {a \in A} q ^ {P, \pi_ {t}} (s, a) \sum_ {s ^ {\prime} \in S _ {k (s) + 1}} \left(\widehat {P} _ {t} \left(s ^ {\prime} | s, a\right) - P \left(s ^ {\prime} | s, a\right)\right) \left(V ^ {\widehat {P} _ {t}, \pi^ {\star}} \left(s ^ {\prime}; \alpha_ {t}\right) - V ^ {\widehat {P} _ {t}, \pi_ {t}} \left(s ^ {\prime}; \alpha_ {t}\right)\right) \\ \leq 2 \sum_ {s \neq s _ {L}} \sum_ {a \in A} \sum_ {s ^ {\prime} \in S _ {k (s) + 1}} q ^ {P, \pi_ {t}} (s, a) B _ {i (t)} (s, a, s ^ {\prime}) \left| V ^ {\widehat {P} _ {t}, \pi^ {*}} \left(s ^ {\prime}; \alpha_ {t}\right) - V ^ {\widehat {P} _ {t}, \pi_ {t}} \left(s ^ {\prime}; \alpha_ {t}\right) \right| \\ \leq \mathcal {O} \left(\sum_ {k = 0} ^ {L - 1} \sum_ {(s, a, s ^ {\prime}) \in W _ {k}} q ^ {P, \pi_ {t}} (s, a) \sqrt {\frac {P \left(s ^ {\prime} \mid s , a\right) \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}} \cdot \left| V ^ {\widehat {P} _ {t}, \pi^ {*}} \left(s ^ {\prime}; \alpha_ {t}\right) - V ^ {\widehat {P} _ {t}, \pi_ {t}} \left(s ^ {\prime}; \alpha_ {t}\right) \right|\right) \\ + \mathcal {O} \left(L | S | \sum_ {s \neq s _ {L}} \sum_ {a \in A} q ^ {P, \pi_ {t}} (s, a) \left(\frac {\mathcal {C} ^ {\mathsf {P}} + \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}\right)\right), \\ \end{array}
$$
where the last step follows from Lemma D.2.7 and the fact that  $V^{\hat{P}_t,\pi}(s';\alpha_t)\leq L$  for any  $\pi$
By applying Lemma D.5.3, we have the second term bounded by  $\mathcal{O}\left((C^{\mathsf{P}} + \log (\iota))L^{2}|S|^{4}|A|\log (\iota)\right)$ . On the other hand, for the first term, we can bound  $V^{\widehat{P}_t,\pi^\star}(s';\alpha_t) - V^{\widehat{P}_t,\pi_t}(s';\alpha_t)$  as
$$
\begin{array}{l} \left| V ^ {\widehat {P} _ {t}, \pi^ {*}} \left(s ^ {\prime}; \alpha_ {t}\right) - V ^ {\widehat {P} _ {t}, \pi_ {t}} \left(s ^ {\prime}; \alpha_ {t}\right) \right| \\ \leq \sum_ {u \neq s _ {L}} \sum_ {v \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (u | s ^ {\prime}) | \pi_ {t} (v | u) - \pi^ {\star} (v | u) | Q ^ {\widehat {P} _ {t}, \pi^ {\star}} (u, v; \alpha_ {t}) \\ \leq L \sum_ {u \in S} \sum_ {v \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (u | s ^ {\prime}) | \pi_ {t} (v | u) - \pi^ {\star} (v | u) | \\ \leq \mathcal {O} \left(L \sum_ {k = k (s ^ {\prime})} ^ {L - 1} \sum_ {u \in S _ {k}} \sum_ {v \neq \pi^ {\star} (u)} q ^ {\widehat {P} _ {t}, \pi_ {t}} (u, v | s ^ {\prime})\right) \\ \leq \mathcal {O} \left(L \sum_ {k = k (s ^ {\prime})} ^ {L - 1} \sum_ {u \in S _ {k}} \sum_ {v \neq \pi^ {*} (u)} q ^ {P, \pi_ {t}} (u, v | s ^ {\prime})\right) \\ \end{array}
$$
where the first step follows from Lemma D.3.2; the second step follows from the fact that  $Q^{\widehat{P}_t,\pi^{\star}}(u,v;\alpha_t)\in [0,L]$ ; the third step uses the same reasoning as the proof of Corollary D.3.5; and the last step uses Corollary C.8.2.
Finally, we consider the following term
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} \sum_ {h = 0} ^ {L - 1} \sum_ {(s, a, s ^ {\prime}) \in W _ {h}} q ^ {P, \pi_ {t}} (s, a) \sqrt {\frac {P (s ^ {\prime} | s , a) \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}} \sum_ {k = h + 1} ^ {L - 1} \sum_ {u \in S _ {k}} \sum_ {v \neq \pi^ {\star} (u)} q ^ {P, \pi_ {t}} (u, v | s ^ {\prime}) \\ = \sum_ {t = 1} ^ {T} \sum_ {h = 0} ^ {L - 1} \sum_ {(s, a, s ^ {\prime}) \in W _ {h}} \sum_ {k = h + 1} ^ {L - 1} \sum_ {u \in S _ {k}} \sum_ {v \neq \pi^ {\star} (u)} \sqrt {\frac {q ^ {P , \pi_ {t}} (s , a) q ^ {P , \pi_ {t}} (u , v | s ^ {\prime}) \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}} \\ \cdot \sqrt {q ^ {P , \pi_ {t}} (s , a) P (s ^ {\prime} | s , a) q ^ {P , \pi_ {t}} (u , v | s ^ {\prime})} \\ \leq \sum_ {h = 0} ^ {L - 1} \sqrt {\sum_ {t = 1} ^ {T} \sum_ {(s , a , s ^ {\prime}) \in W _ {h}} \sum_ {k = h + 1} ^ {L - 1} \sum_ {u \in S _ {k}} \sum_ {v \neq \pi^ {\star} (u)} \frac {q ^ {P , \pi_ {t}} (s , a) q ^ {P , \pi_ {t}} (u , v | s ^ {\prime}) \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}} \\ \cdot \sqrt {\sum_ {t = 1} ^ {T} \sum_ {(s , a , s ^ {\prime}) \in W _ {h}} \sum_ {k = h + 1} ^ {L - 1} \sum_ {u \in S _ {k}} \sum_ {v \neq \pi^ {\star} (u)} q ^ {P , \pi_ {t}} (s , a) P \left(s ^ {\prime} | s , a\right) q ^ {P , \pi_ {t}} (u , v | s ^ {\prime})} \\ \leq \sum_ {h = 0} ^ {L - 1} \sqrt {\left| S _ {h + 1} \right| L \sum_ {t = 1} ^ {T} \sum_ {s \in S _ {h}} \sum_ {a \in A} \frac {q ^ {P , \pi_ {t}} (s , a) \log (\iota)}{\widehat {m} _ {i (t)} (s , a)}} \cdot \sqrt {\sum_ {t = 1} ^ {T} \sum_ {u \in S} \sum_ {v \neq \pi^ {\star} (u)} q ^ {P , \pi_ {t}} (u , v)} \\ \end{array}
$$
$$
\begin{array}{l} \leq \left(\sum_ {h = 0} ^ {L - 1} \sqrt {L \left| S _ {h + 1} \right| \left| S _ {h} \right| \left| A \right| \log^ {2} (\iota)}\right) \cdot \sqrt {\sum_ {t = 1} ^ {T} \sum_ {u \in S} \sum_ {v \neq \pi^ {\star} (u)} q ^ {P , \pi_ {t}} (u , v)} \\ = \mathcal {O} \left(\sqrt {L | S | ^ {2} | A | \log^ {2} (\iota) \sum_ {t = 1} ^ {T} \sum_ {s \neq s _ {L}} \sum_ {a \neq \pi^ {\star} (a)} q ^ {P , \pi_ {t}} (s , a)}\right) \\ = \mathcal {O} \left(\sqrt {\underbrace {L | S | ^ {2} | A | \log^ {2} (\iota) \sum_ {t = 1} ^ {T} \sum_ {s \neq s _ {L}} \sum_ {a \neq \pi^ {\star} (a)} q ^ {P _ {t} , \pi_ {t}} (s , a)}} + \sqrt {C ^ {\mathsf {P}} \cdot L ^ {2} | S | ^ {2} | A | \log^ {2} (\iota)}\right), \\ \end{array}
$$
where the second step applies Cauchy-Schwarz inequality; the third step follows from the fact that  $\sum_{s\in S_k}\sum_{a\in A}\sum_{s'\in S_{k(s) + 1}}q^{P,\pi_t}(s,a)P(s'|s,a)q^{P,\pi_t}(u,v|s') = q^{P,\pi_t}(u,v)$ ; the fourth step follows from Lemma D.5.3 conditioning on the event  $\mathcal{E}_{\mathrm{EST}}$ ; the fifth step uses the fact that  $\sqrt{x + y} \leq \sqrt{x} + \sqrt{y}$  for  $x, y \geq 0$ ; the last step follows from Corollary D.3.6.
C.7 Proof of Lemma C.3.4
In this section we bound ESTREG using a learning rate that depends on  $t$  and  $(s, a)$ , which is crucial to obtain a self-bounding quantity. Another key observation is that the estimated transition function is constant within each epoch, so we first bound ESTREG within one epoch before summing them.
Recall that  $E_{i}$  is a set of episodes that belong to epoch  $i$  and  $N$  is the total number of epochs through  $T$  episodes. By using the fact that  $\widehat{P}_{t} = \widetilde{P}_{i}$  for episode  $t$  belonging to epoch  $i$ , we make the following decomposition  $\mathrm{ESTREG}(\pi^{\star})$
$$
\begin{array}{l} \operatorname {E S T R E G} \left(\pi^ {\star}\right) = \mathbb {E} \left[ \sum_ {i = 1} ^ {N} \sum_ {t \in E _ {i}} \left\langle q ^ {\widetilde {P} _ {i}, \pi_ {t}} - q ^ {\widehat {P} _ {t}, \pi^ {\star}}, \widehat {\ell} _ {t} - b _ {t} \right\rangle \right] \\ \leq \mathbb {E} \left[ \sum_ {i = 1} ^ {N} \mathbb {E} _ {t _ {i}} \left[ \sum_ {t \in E _ {i}} \left\langle q ^ {\widetilde {P} _ {i}, \pi_ {t}} - q ^ {\widehat {P} _ {t}, \pi^ {\star}}, \widehat {\ell} _ {t} - b _ {t} \right\rangle \right] \right] = \mathbb {E} \left[ \sum_ {i = 1} ^ {N} \operatorname {E S T R E G} _ {i} (\pi^ {\star}) \right]. \\ \end{array}
$$
This learning rate is defined in Definition 5.2 and restated below.
Definition C.7.1. For any  $t$ , if it is the starting episode of an epoch, we set  $\gamma_t(s,a) = 256L^2 |S|$ ; otherwise, we set
$$
\gamma_ {t + 1} (s, a) = \gamma_ {t} (s, a) + \frac {D \nu_ {t} (s , a)}{2 \gamma_ {t} (s , a)},
$$
where  $D = \frac{1}{\log(\iota)}$  and
$$
\nu_ {t} (s, a) = q ^ {\widetilde {P} _ {i (t)}, \pi_ {t}} (s, a) ^ {2} \left(Q ^ {\widetilde {P} _ {i (t)}, \pi_ {t}} (s, a; \widehat {\ell} _ {t}) - V ^ {\widetilde {P} _ {i (t)}, \pi_ {t}} (s; \widehat {\ell} _ {t})\right) ^ {2}. \tag {35}
$$
Importantly, both  $Q^{\widetilde{P}_{i(t)},\pi_t}(s,a;\widehat{\ell}_t)$  and  $V^{\widetilde{P}_{i(t)},\pi_t}(s;\widehat{\ell}_t)$  can be computed, which ensures that the learning rate is properly defined.
C.7.1 Properties of the Learning Rate
In this section, we prove key properties of  $\nu_{t}(s,a)$  and of  $\gamma_t(s,a)$ . We first present some results in Lemma C.7.2 that are useful to bound  $\nu_{t}(s,a)$ , and then use these results to bound  $\gamma_t(s,a)$ .
Lemma C.7.2. For any state-action pair  $(s, a)$  and any episode  $t$ , it holds that
$$
q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a) Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t}) \leq L, a n d q ^ {\widehat {P} _ {t}, \pi_ {t}} (s) V ^ {\widehat {P} _ {t}, \pi_ {t}} (s; \widehat {\ell} _ {t}) \leq L, \forall (s, a) \in S \times A,
$$
which ensures  $q^{\widehat{P}_t,\pi_t}(s,a)\left(Q^{\widehat{P}_t,\pi_t}(s,a;\widehat{\ell}_t) - V^{\widehat{P}_t,\pi_t}(s;\widehat{\ell}_t)\right)\in [-L,L]$ . Suppose that the high-probability event  $\mathcal{E}_{\mathrm{CON}}$  holds. Then, it further holds for all state-action pair  $(s,a)$  that
$$
q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a) ^ {2} \cdot \mathbb {E} _ {t} \left[ \left(Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t}) - V ^ {\widehat {P} _ {t}, \pi_ {t}} (s; \widehat {\ell} _ {t})\right) ^ {2} \right] \leq \mathcal {O} \left(L ^ {2} q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a) \left(1 - \pi_ {t} (a | s)\right) + L | S | C _ {t} ^ {\mathcal {P}}\right),
$$
where  $\widehat{P}_t$  here is the optimistic transition  $\widetilde{P}_{i(t)}$  defined in Definition C.1.1.
Proof. We first verify  $q^{\widehat{P}_t, \pi_t}(s, a) Q^{\widehat{P}_t, \pi_t}(s, a; \widehat{\ell_t}) \leq L$ :
$$
\begin{array}{l} q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a) Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t}) \\ = q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a) \sum_ {h = k (s)} \sum_ {u \in S} \sum_ {v \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (u, v | s, a) \widehat {\ell} _ {t} (u, v) \\ = q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a) \sum_ {h = k (s)} \sum_ {u \in S} \sum_ {v \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (u, v | s, a) \cdot \frac {\mathbb {I} _ {t} (u , v) \ell_ {t} (u , v)}{u _ {t} (u , v)} \\ \leq \sum_ {h = k (s)} \sum_ {u \in S _ {h}} \sum_ {v \in A} \mathbb {I} _ {t} (u, v) \cdot \frac {q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a) q ^ {\widehat {P} _ {t} , \pi_ {t}} (u , v | s , a)}{u _ {t} (u , v)} \\ \leq \sum_ {h = k (s)} \sum_ {u \in S _ {h}} \sum_ {v \in A} \mathbb {I} _ {t} (u, v) \leq L, \\ \end{array}
$$
where the second step follows from the definition of loss estimator, the fourth step follows from  $q^{\widehat{P}_t,\pi_t}(s,a)q^{\widehat{P}_t,\pi_t}(u,v|s,a)\leq q^{\widehat{P}_t,\pi_t}(u,v)\leq u_t(u,v)$ , and the last step uses the fact that  $\sum_{u\in S_h}\sum_{v\in A}\mathbb{I}_t(u,v) = 1$ . Following the same idea, we can show that  $q^{\widehat{P}_t,\pi_t}(s)V^{\widehat{P}_t,\pi_t}(s;\widehat{\ell_t})\leq L$  as well.
Next, we have  $\mathbb{E}_t\left[\left(Q^{\widehat{P}_t,\pi_t}(s,a;\widehat{\ell}_t) - V^{\widehat{P}_t,\pi_t}(s;\widehat{\ell}_t)\right)^2\right]$  bounded as
$$
\begin{array}{l} \mathbb {E} _ {t} \left[ \left(Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t}) - V ^ {\widehat {P} _ {t}, \pi_ {t}} (s; \widehat {\ell} _ {t})\right) ^ {2} \right] \\ = \mathbb {E} _ {t} \left[ \left(Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t}) - \pi_ {t} (a | s) Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t}) - \sum_ {a ^ {\prime} \neq a} \pi_ {t} (a ^ {\prime} | s) Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a ^ {\prime}; \widehat {\ell} _ {t})\right) ^ {2} \right] \\ \leq 2 \cdot \mathbb {E} _ {t} \left[ \left(Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t}) - \pi_ {t} (a | s) Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t})\right) ^ {2} \right] \\ + 2 \cdot \mathbb {E} _ {t} \left[ \left(\sum_ {a ^ {\prime} \neq a} \pi_ {t} \left(a ^ {\prime} \mid s\right) Q ^ {\widehat {P} _ {t}, \pi_ {t}} \left(s, a ^ {\prime}; \widehat {\ell} _ {t}\right)\right) ^ {2} \right] \\ = 2 \left(1 - \pi_ {t} (a | s)\right) ^ {2} \mathbb {E} _ {t} \left[ \left(Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t})\right) ^ {2} \right] + 2 \mathbb {E} _ {t} \left[ \left(\sum_ {a ^ {\prime} \neq a} \pi_ {t} \left(a ^ {\prime} | s\right) Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a ^ {\prime}; \widehat {\ell} _ {t})\right) ^ {2} \right], \tag {36} \\ \end{array}
$$
where the second step follows from the fact that  $(x + y)^2 \leq 2(x^2 + y^2)$  for any  $x, y \in \mathbb{R}$ . By direct calculation, we have
$$
\begin{array}{l} \mathbb {E} _ {t} \left[ \left(Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, a; \widehat {\ell} _ {t})\right) ^ {2} \right] \\ = \mathbb {E} _ {t} \left[ \left(\sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) \widehat {\ell} _ {t} (x, y)\right) ^ {2} \right] \\ \leq L \cdot \mathbb {E} _ {t} \left[ \sum_ {k = k (s)} ^ {L - 1} \left(\sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) \widehat {\ell} _ {t} (x, y)\right) ^ {2} \right] \\ = L \cdot \mathbb {E} _ {t} \left[ \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) ^ {2} \widehat {\ell} _ {t} (x, y) ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} \leq L \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) ^ {2} \cdot \left(\frac {q ^ {P _ {t} , \pi_ {t}} (x , y)}{u _ {t} (x , y) ^ {2}}\right) \\ = \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)} \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) \left(\frac {q ^ {\widehat {P} _ {t} , \pi_ {t}} (x , y | s , a) q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)}{u _ {t} (s , a)}\right) \cdot \left(\frac {q ^ {P _ {t} , \pi_ {t}} (x , y)}{u _ {t} (s , a)}\right) \\ \leq \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)} \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) \left(\frac {q ^ {P _ {t} , \pi_ {t}} (x , y)}{u _ {t} (s , a)}\right) \\ \leq \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)} \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) \left(\frac {q ^ {P , \pi_ {t}} (x , y)}{u _ {t} (x , y)} + \frac {C _ {t} ^ {\mathbf {P}}}{u _ {t} (x , y)}\right), \tag {37} \\ \end{array}
$$
where the second step uses Cauchy-Schwarz inequality; the third step uses the fact that  $\widehat{\ell}_t(s,a)\cdot \widehat{\ell}_t(s',a') = 0$  for any  $(s,a)\neq (s',a')$ ; the fourth step takes the conditional expectation of  $\widehat{\ell}_t(x,y)^2$ ; the sixth step follows from the fact that  $q^{\widehat{P}_t,\pi_t}(x,y|s,a)q^{\widehat{P}_t,\pi_t}(s,a)\leq q^{\widehat{P}_t,\pi_t}(x,y)\leq u_t(x,y)$  according to the definition of upper occupancy bound; the last step follows from Corollary D.3.6.
Similarly, for the second term in Eq. (36), we have
$$
\begin{array}{l} \mathbb {E} _ {t} \left[ \left(\sum_ {b \neq a} \pi_ {t} (b | s) Q ^ {\widehat {P} _ {t}, \pi_ {t}} (s, b; \widehat {\ell_ {t}})\right) ^ {2} \right] \\ \leq L \cdot \mathbb {E} _ {t} \left[ \sum_ {k = k (s)} ^ {L - 1} \left(\sum_ {x \in S _ {k}} \sum_ {y \in A} \left(\sum_ {b \neq a} \pi_ {t} (b | s) q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, b)\right) \widehat {\ell} _ {t} (x, y)\right) ^ {2} \right] \\ = L \cdot \mathbb {E} _ {t} \left[ \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} \left(\sum_ {b \neq a} \pi_ {t} (b | s) q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, b)\right) ^ {2} \widehat {\ell} _ {t} (x, y) ^ {2} \right] \\ \leq L \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} \left(\sum_ {b \neq a} \pi_ {t} (b | s) q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, b)\right) ^ {2} \left(\frac {q ^ {P _ {t} , \pi_ {t}} (x , y)}{u _ {t} (x , y) ^ {2}}\right) \\ \leq \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s)} \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} \left(\sum_ {b \neq a} \pi_ {t} (b | s) q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, b)\right) \left(\frac {q ^ {P _ {t} , \pi_ {t}} (x , y)}{u _ {t} (x , y)}\right) \\ \leq \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s)} \sum_ {b \neq a} \pi_ {t} (b | s) \left(\sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, b) \left(\frac {q ^ {P , \pi_ {t}} (x , y)}{u _ {t} (x , y)} + \frac {C _ {t} ^ {\mathbf {P}}}{u _ {t} (x , y)}\right)\right), \tag {38} \\ \end{array}
$$
where the first three steps are following the same idea of previous analysis; the fourth step uses the fact that  $\sum_{b\neq a}q^{\widehat{P}_t,\pi_t}(s)\pi_t(b|s)q^{\widehat{P}_t,\pi_t}(x,y|s,b)\leq q^{\widehat{P}_t,\pi_t}(x,y)$ ; the last step follows from Corollary D.3.6 as well.
Conditioning on the event  $\mathcal{E}_{\mathrm{CON}}$ , we have the term in Eq. (37) further bounded as
$$
\begin{array}{l} \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)} \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) \left(\frac {q ^ {P , \pi_ {t}} (x , y)}{u _ {t} (x , y)} + \frac {C _ {t} ^ {\mathsf {P}}}{u _ {t} (x , y)}\right) \\ \leq \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)} \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) \left(1 + \frac {C _ {t} ^ {\mathbf {P}}}{u _ {t} (x , y)}\right) \\ = \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)} \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) \\ \end{array}
$$
$$
\begin{array}{l} + \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)} \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, a) \left(\frac {C _ {t} ^ {\mathsf {P}}}{u _ {t} (x , y)}\right) \\ \leq \frac {L ^ {2}}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)} + \frac {L C _ {t} ^ {\mathsf {P}}}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s , a)} \cdot \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} \left(\frac {q ^ {\widehat {P} _ {t} , \pi_ {t}} (x , y | s , a)}{u _ {t} (x , y)}\right), \tag {39} \\ \end{array}
$$
where the second step follows from the fact that  $q^{P,\pi_t}(x,y) \leq u_t(x,y)$  as  $P \in \mathcal{P}_{i(t)}$  according to the event  $\mathcal{E}_{\mathrm{CON}}$ ; the last step follows from the fact that  $\sum_{x \in S_k} \sum_{y \in A} q^{\widetilde{P}_t, \pi_t}(x,y|s,a) \leq 1$ .
Following the same argument, for the term in Eq. (38), we have
$$
\begin{array}{l} \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s)} \sum_ {b \neq a} \pi_ {t} (b | s) \left(\sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} q ^ {\widehat {P} _ {t}, \pi_ {t}} (x, y | s, b) \left(\frac {q ^ {P , \pi_ {t}} (x , y)}{u _ {t} (x , y)} + \frac {C _ {t} ^ {\mathsf {P}}}{u _ {t} (x , y)}\right)\right) \\ \leq \frac {L}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s)} \sum_ {b \neq a} \pi_ {t} (b | s) \left(L + \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} \left(\frac {q ^ {\widehat {P} _ {t} , \pi_ {t}} (x , y | s , b) C _ {t} ^ {\mathbf {P}}}{u _ {t} (x , y)}\right)\right) \\ \leq \frac {L ^ {2} \left(1 - \pi_ {t} (a | s)\right)}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s)} + \frac {L C _ {t} ^ {\mathbf {P}}}{q ^ {\widehat {P} _ {t} , \pi_ {t}} (s)} \sum_ {b \neq a} \pi_ {t} (b | s) \sum_ {k = k (s)} ^ {L - 1} \sum_ {x \in S _ {k}} \sum_ {y \in A} \left(\frac {q ^ {\widehat {P} _ {t} , \pi_ {t}} (x , y | s , b)}{u _ {t} (x , y)}\right). \tag {40} \\ \end{array}
$$
Plugging Eq. (39) and Eq. (40) into Eq. (36), we have