SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\left(\frac {1}{K H} + \widehat {d} _ {h} ^ {\pi^ {(k)}} (s , a)\right) \widehat {d} _ {h} ^ {\mathrm {o f f}} (s , a)}{\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) ^ {2}} \right] \\ \geq \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\left(\frac {1}{K H} + \mathbb {E} _ {\pi \sim \mu^ {\text {o f f}}} \left[ \widehat {d} _ {h} ^ {\pi} (s , a) \right]\right) \widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) ^ {2}} \right] \\ \geq \frac {1}{1 2} \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\left(\widehat {d} _ {h} ^ {\mathrm {o f f}} (s , a)\right) ^ {2}}{\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) ^ {2}} \right] \\ \geq 3 \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \right] \mathbb {1} \left(\frac {\widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} > 3 6\right) \\ = 3 \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \right] \\ - 3 \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \right] \mathbb {1} \left(\frac {\widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \leq 3 6\right) \\ \geq 3 \sum_ {h = 1} ^ {H} \sum_ {s \in S} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \right] - 1 0 8 S H \\ \geq 2 \sum_ {h = 1} ^ {H} \sum_ {s \in S} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \right], \tag {90} \\ \end{array}
$$
where the first inequality follows from the choice (32) of  $\pi^{(k)}$ , the second inequality is a consequence of the relation (87), and the last line makes use of (89). This in turn allows one to demonstrate that
$$
\begin{array}{l} \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\left(\widehat {d} _ {h} ^ {\pi^ {(k)}} (s , a) - \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) \widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) ^ {2}} \right] \\ = \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\left(\frac {1}{K H} + \widehat {d} _ {h} ^ {\pi^ {(k)}} (s , a)\right) \widehat {d} _ {h} ^ {\mathrm {o f f}} (s , a)}{\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) ^ {2}} \right] - \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) \widehat {d} _ {h} ^ {\mathrm {o f f}} (s , a)}{\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) ^ {2}} \right] \\ \geq \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\widehat {d} _ {h} ^ {\mathrm {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \right] > 1 0 8 S H, \\ \end{array}
$$
where the last line results from (90) and the condition (89). We can then readily derive
$$
\sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\widehat {d} _ {h} ^ {\mathrm {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \right] - \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\widehat {d} _ {h} ^ {\mathrm {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\bar {\pi} ^ {\prime} \sim \mu^ {(k + 1)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \right]
$$
$$
\begin{array}{l} = \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \right. \frac {\left(\mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k + 1)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right] - \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) \widehat {d} _ {h} ^ {\mathrm {o f f}} (s , a)}{\left( \right.\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k + 1)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right)} \left. \right] \\ = \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \right. \frac {\alpha \left(\widehat {d} _ {h} ^ {\pi^ {(k)}} (s , a) - \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) \widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\left( \right.\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k + 1)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right)} \left. \right] \\ = \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\alpha \left(\widehat {d} _ {h} ^ {\pi^ {(k)}} (s , a) - \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) \widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) ^ {2}} \right] \\ - \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\alpha^ {2} \left(\widehat {d} _ {h} ^ {\pi^ {(k)}} (s , a) - \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) ^ {2} \widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right) ^ {2} \left(\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu^ {(k + 1)}} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]\right)} \right] \\ \geq 1 0 8 \alpha S H - \alpha^ {2} \left(K ^ {3} H ^ {4}\right) = \frac {1 0 8 S ^ {2}}{K ^ {3} H ^ {2}} - \frac {S ^ {2}}{K ^ {3} H ^ {2}} = \frac {1 0 7 S ^ {2}}{K ^ {3} H ^ {2}}, \tag {91} \\ \end{array}
$$
where the third line relies on the update rule (35), and the last line utilizes the choice (36) of  $\alpha$ .
In summary, the above argument reveals that: before the stopping criterion is met, each iteration is able to make progress at least as large as in (91). Recognizing the crude bound
$$
0 \leq \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \frac {\widehat {d} _ {h} ^ {\text {o f f}} (s , a)}{\frac {1}{K H} + \mathbb {E} _ {\pi^ {\prime} \sim \mu} \left[ \widehat {d} _ {h} ^ {\pi^ {\prime}} (s , a) \right]} \right] \leq K H \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \mathbb {E} _ {a \sim \pi_ {h} ^ {t + 1} (\cdot | s)} \left[ \widehat {d} _ {h} ^ {\text {o f f}} (s, a) \right] \leq K H ^ {2}
$$
that holds for any  $\mu \in \Delta (\Pi)$ , one can combine this with (91) to conclude that: the proposed procedure terminates within  $O\left(\frac{K^4H^4}{S^2}\right)$  iterations, as claimed.
G Proofs of technical lemmas
G.1 Proof of Lemma 2
The Bernstein inequality combined with the union bound tells us that, with probability at least  $1 - \delta / 3$ ,
$$
\begin{array}{l} \left| \frac {2 N _ {h} ^ {\text {o f f}} (s , a)}{K ^ {\text {o f f}}} - d _ {h} ^ {\text {o f f}} (s, a) \right| \leq 6 \sqrt {\frac {d _ {h} ^ {\text {o f f}} (s , a) \log \frac {H S A}{\delta}}{K ^ {\text {o f f}}}} + \frac {6 \log \frac {H S A}{\delta}}{K ^ {\text {o f f}}} \\ \leq \frac {d _ {h} ^ {\text {o f f}} (s , a)}{2} + \frac {2 4 \log \frac {H S A}{\delta}}{K ^ {\text {o f f}}} \tag {92} \\ \end{array}
$$
holds simultaneously for all  $(s, a, h) \in S \times \mathcal{A} \times [H]$ , where the last line invokes the AM-GM inequality. This in turn reveals that
$$
\frac {4 N _ {h} ^ {\text {o f f}} (s , a)}{3 K ^ {\text {o f f}}} - \frac {1 6 \log \frac {H S A}{\delta}}{K ^ {\text {o f f}}} \leq d _ {h} ^ {\text {o f f}} (s, a) \leq \frac {4 N _ {h} ^ {\text {o f f}} (s , a)}{K ^ {\text {o f f}}} + \frac {4 8 \log \frac {H S A}{\delta}}{K ^ {\text {o f f}}}. \tag {93}
$$
As a result, we can show that:
- If  $\frac{N_h^{\mathrm{off}}(s,a)}{K^{\mathrm{off}}} \geq c_{\mathrm{off}}\left(\frac{\log\frac{HSA}{\delta}}{K^{\mathrm{off}}} + \frac{H^4S^4A^4\log\frac{HSA}{\delta}}{N} + \frac{SA}{K^{\mathrm{on}}}\right)$  for some  $c_{\mathrm{off}} \geq 48$ , then one has
$$
\begin{array}{l} \frac {2 N _ {h} ^ {\text {o f f}} (s , a)}{3 K ^ {\text {o f f}}} \leq d _ {h} ^ {\text {o f f}} (s, a) \leq \frac {6 N _ {h} ^ {\text {o f f}} (s , a)}{K ^ {\text {o f f}}} \quad \text {a n d} \quad \widehat {d} _ {h} ^ {\text {o f f}} (s, a) = \frac {2 N _ {h} ^ {\text {o f f}} (s , a)}{K ^ {\text {o f f}}} (94a) \\ \Longrightarrow \quad \frac {1}{3} \widehat {d} _ {h} ^ {\text {o f f}} (s, a) \leq d _ {h} ^ {\text {o f f}} (s, a) \leq 3 \widehat {d} _ {h} ^ {\text {o f f}} (s, a). (94b) \\ \end{array}
$$
- If instead  $\frac{N_h^{\mathrm{off}}(s,a)}{K^{\mathrm{off}}} < c_{\mathrm{off}}\left(\frac{\log\frac{HSA}{\delta}}{K^{\mathrm{off}}} + \frac{H^4S^4A^4\log\frac{HSA}{\delta}}{N} + \frac{SA}{K^{\mathrm{on}}}\right)$ , then one has  $\widehat{d}_h^{\mathrm{off}}(s,a) = 0$  and therefore,
$$
d _ {h} ^ {\text {o f f}} (s, a) \geq 0 = \frac {1}{3} \widehat {d} _ {h} ^ {\text {o f f}} (s, a),
$$
$$
\begin{array}{l} d _ {h} ^ {\text {o f f}} (s, a) \leq \frac {4 N _ {h} ^ {\text {o f f}} (s , a)}{K ^ {\text {o f f}}} + \frac {4 8 \log \frac {H S A}{\delta}}{K ^ {\text {o f f}}} <   5 c _ {\text {o f f}} \left\{\frac {\log \frac {H S A}{\delta}}{K ^ {\text {o f f}}} + \frac {H ^ {4} S ^ {4} A ^ {4} \log \frac {H S A}{\delta}}{N} + \frac {S A}{K ^ {\text {o n}}} \right\} \\ = \widehat {d} _ {h} ^ {\text {o f f}} (s, a) + 5 c _ {\text {o f f}} \left\{\frac {\log \frac {H S A}{\delta}}{K ^ {\text {o f f}}} + \frac {H ^ {4} S ^ {4} A ^ {4} \log \frac {H S A}{\delta}}{N} + \frac {S A}{K ^ {\text {o n}}} \right\}. \\ \end{array}
$$
Taken collectively, these inequalities demonstrate that
$$
\frac {1}{3} \widehat {d} _ {h} ^ {\text {o f f}} (s, a) \leq d _ {h} ^ {\text {o f f}} (s, a) \leq \widehat {d} _ {h} ^ {\text {o f f}} (s, a) + 5 c _ {\text {o f f}} \left\{\frac {\log \frac {H S A}{\delta}}{K ^ {\text {o f f}}} + \frac {H ^ {4} S ^ {4} A ^ {4} \log \frac {H S A}{\delta}}{N} + \frac {S A}{K ^ {\text {o n}}} \right\}, \tag {95}
$$
provided that  $c_{\mathrm{off}} > 0$  is sufficiently large.
G.2 Proof of Lemma 6
Before embarking on the proof, let us introduce several notation. For each  $1 \leq h \leq H$ , define  $P_h^{\pi^*} \in \mathbb{R}^{S \times S}$  and  $d_h^{\pi^*} \in \mathbb{R}^S$  such that: for all  $s \in S$ ,
$$
P _ {h} ^ {\pi^ {\star}} (s, \cdot) = P _ {h} \left(\cdot | s, \pi^ {\star} (s)\right) \quad \text {a n d} \quad d _ {h} ^ {\pi^ {\star}} (s) = d _ {h} ^ {\pi^ {\star}} \left(s, \pi^ {\star} (s)\right). \tag {96}
$$
To begin with, it can be easily seen from (69) and the basic fact  $\mathsf{Var}_{P_h(\cdot |s,a)}\bigl (\widehat{V}_{h + 1}\bigr)\leq H^2$  that
$$
\begin{array}{l} \left\langle d_{j}^{\pi^{\star}},V_{j}^{\star} - V_{j}^{\widehat{\pi}}\right\rangle \leq \underbrace{8H\sum_{h:h\geq j}\sum_{s}\max_{a:(s,a)\in\mathcal{I}_{h}}\sqrt{2d_{h}^{\star}(s,a)C^{\star}(\sigma)\widehat{d}_{h}^{\mathrm{off}}(s,a)}\sqrt{\frac{c_{b}\log\frac{K}{\delta}}{N_{h}^{\mathrm{trim}}(s,a)+1}}}_{=: \gamma_{3}} \\ + \underbrace {4 H \sum_ {h : h \geq j} \sum_ {(s , a) \in \mathcal {T} _ {h}} \widehat {d} _ {h} ^ {\pi^ {*}} (s , a) \sqrt {\frac {c _ {\mathrm {b}} \log \frac {K}{\delta}}{N _ {h} ^ {\mathrm {t r i m}} (s , a) + 1}}} _ {=: \gamma_ {2}} + \frac {6 1 c _ {\xi} H ^ {6} S ^ {4} A ^ {4}}{N} \log \frac {K}{\delta} \tag {97} \\ \end{array}
$$
holds for any  $j \in [H]$ . Note that we have bounded  $\gamma_{2}$  in (79). We then need to bound  $\gamma_{3}$ .
With regards to the term  $\gamma_{3}$ : invoking similar arguments as for (72) leads to
$$
\begin{array}{l} \gamma_ {3} \leq 6 4 c _ {\mathrm {b}} H \sum_ {h: h \geq j} \sum_ {s} \max  _ {a} \sqrt {2 d _ {h} ^ {\pi^ {\star}} (s , a) C ^ {\star} (\sigma) \widehat {d} _ {h} ^ {\text {o f f}} (s , a)} \sqrt {\frac {\log \frac {K}{\delta}}{1 / H + \frac {1}{7 2} K ^ {\text {o n}} \mathbb {E} _ {\pi \in \mu^ {\text {i m i t a t e}}} \left[ \widehat {d} _ {h} ^ {\pi} (s , a) \right]}} + 4 H ^ {2} S C ^ {\star} (\sigma) \Big (\frac {\xi}{N} + \frac {1}{K ^ {\text {o n}} H} \Big) \\ \leq 64c_{\mathsf{b}}H\sqrt{2\sum_{h:h\geq j}\sum_{s,a}d_{h}^{\pi^{\star}}(s,a)}\cdot \sqrt{\sum_{h:h\geq j}\sum_{s}\max_{a}\frac{C^{\star}(\sigma)\widehat{d}_{h}^{\text{off}}(s,a)\log\frac{K}{\delta}}{1 / H + \frac{1}{72}K^{\text{on}}\mathbb{E}_{\pi\in\mu^{\text{imitate}}}\left[\widehat{d}_{h}^{\pi}(s,a)\right}}} + 4H^{2}SC^{\star}(\sigma)\Bigl(\frac{\xi}{N} +\frac{1}{K^{\text{on}}H}\Bigr) \\ \leq 7 6 8 c _ {\mathrm {b}} \sqrt {\frac {1 3 H ^ {4} S C ^ {\star} (\sigma)}{K ^ {\mathrm {o n}}} \log \frac {K}{\delta}} + 4 H ^ {2} S C ^ {\star} (\sigma) \left(\frac {\xi}{N} + \frac {1}{K ^ {\mathrm {o n}} H}\right), \tag {98} \\ \end{array}
$$
where the second step invokes the Cauchy-Schwartz inequality, and the last line comes from (67). Similarly, repeating the above argument but focusing on the offline dataset, we can derive (which we omit for the sake of brevity)
$$
\begin{array}{l} \gamma_{3}\leq 64c_{\mathsf{b}}H\sum_{h:h\geq j}\sum_{s}\max_{a}\sqrt{2d_{h}^{\pi^{\star}}(s,a)C^{\star}(\sigma)\widehat{d}_{h}^{\text{off}}(s,a)}\sqrt{\frac{\log\frac{K}{\delta}}{1 / H + \left.\frac{1}{72}K^{\text{off}}d_{h}^{\text{off}}(s,a)\right]}} +4H^{2}SC^{\star}(\sigma)\Bigl(\frac{\xi}{N} +\frac{1}{K^{\text{off}}H}\Bigr) \\ \leq 64c_{\mathsf{b}}H\sqrt{2\sum_{h:h\geq j}\sum_{s,a}d_{h}^{\pi^{\star}}(s,a)}\cdot \sqrt{\sum_{h:h\geq j}\sum_{s}\max_{a}\frac{C^{\star}(\sigma)\widehat{d}_{h}^{\text{off}}(s,a)\log\frac{K}{\delta}}{1 / H + \frac{1}{72}K^{\text{off}}d_{h}^{\text{off}}(s,a)}} +4H^{2}SC^{\star}(\sigma)\Bigl(\frac{\xi}{N} +\frac{1}{K^{\text{off}}H}\Bigr) \\ \leq 7 6 8 c _ {\mathrm {b}} \sqrt {\frac {6 H ^ {4} S C ^ {\star} (\sigma)}{K ^ {\text {o f f}}} \log \frac {K}{\delta}} + 4 H ^ {2} S C ^ {\star} (\sigma) \left(\frac {\xi}{N} + \frac {1}{K ^ {\text {o f f}} H}\right), \tag {99} \\ \end{array}
$$
where the last line makes use of (86).
Combining (98), (99) and (79) with (97), we can show that
$$
\begin{array}{l} \left\langle d _ {j} ^ {\pi^ {\star}}, V _ {j} ^ {\star} - V _ {j} ^ {\widehat {\pi}} \right\rangle \leq \min \left\{7 6 8 c _ {\mathrm {b}} \sqrt {\frac {1 3 H ^ {4} S C ^ {\star} (\sigma)}{K ^ {\mathrm {o n}}} \log \frac {K}{\delta}} + 4 H ^ {2} S C ^ {\star} (\sigma) \Big (\frac {\xi}{N} + \frac {1}{K H} \Big), 7 6 8 c _ {\mathrm {b}} \sqrt {\frac {6 H ^ {4} S C ^ {\star} (\sigma)}{K ^ {\mathrm {o f f}}} \log \frac {K}{\delta}} \right\} \\ + 9 6 c _ {\mathrm {b}} \sqrt {\frac {2 H ^ {3} S A (2 \widehat {\sigma} + H S \xi / N)}{K ^ {\mathrm {o n}}} \log \frac {H K}{\delta}} + 4 3 6 H ^ {2} S A \left(\frac {\xi}{N} + \frac {1}{\min \left\{K ^ {\mathrm {o n}} , K ^ {\mathrm {o f f}} \right\} H}\right) \\ + \frac {6 1 c _ {\xi} H ^ {6} S ^ {4} A ^ {4}}{N} \log \frac {K}{\delta} \\ \leq 1 \tag {100} \\ \end{array}
$$
for all  $1 \leq j \leq H$ , with the proviso that
$$
\frac {H ^ {7} S ^ {5} A ^ {4}}{K ^ {\text {o n}}} \log^ {2} \frac {K}{\delta} \leq c _ {1 0}
$$
$$
\frac {H ^ {5} S ^ {4} A ^ {3} C ^ {\star} (\sigma) \log \frac {K}{\delta}}{K ^ {\text {o n}}} \leq c _ {1 0}
$$
$$
\frac {H S C ^ {\star} (\sigma) \log \frac {K}{\delta}}{K ^ {\text {o f f}}} \leq c _ {1 0}
$$
$$
\frac {H S A \log \frac {K}{\delta}}{K ^ {\text {o f f}}} \leq c _ {1 0}
$$
for some sufficiently small constant  $c_{10} > 0$ . As a consequence, we can demonstrate that
$$
\begin{array}{l} \sum_ {h: h \geq j} \sum_ {s, a} d _ {h} ^ {\pi^ {\star}} (s, a) \operatorname {V a r} _ {P _ {h} (\cdot | s, a)} \left(\widehat {V} _ {h + 1}\right) \\ \leq \sum_ {h: h \geq j} 2 \sum_ {s, a} d _ {h} ^ {\pi^ {\star}} (s, a) \left(\operatorname {V a r} _ {P _ {h} (\cdot | s, a)} \left(V _ {h + 1} ^ {\star}\right) + \operatorname {V a r} _ {P _ {h} (\cdot | s, a)} \left(V _ {h + 1} ^ {\star} - \widehat {V} _ {h + 1}\right)\right) \\ \leq 4 H ^ {2} + H \sum_ {h: h \geq j} \sum_ {s} d _ {h} ^ {\pi^ {\star}} (s, \pi^ {\star} (s)) \mathbb {E} _ {P _ {h} (\cdot | s, \pi^ {\star} (s))} [ V _ {h + 1} ^ {\star} - \widehat {V} _ {h + 1} ] \\ = 4 H ^ {2} + H \sum_ {h: h \geq j} \left(d _ {h} ^ {\pi^ {\star}}\right) ^ {\top} P _ {h} ^ {\pi^ {\star}} \left[ V _ {h + 1} ^ {\star} - \widehat {V} _ {h + 1} \right] \\ = 4 H ^ {2} + H \sum_ {h \geq j} \left(d _ {h + 1} ^ {\pi^ {\star}}\right) ^ {\top} \left[ V _ {h + 1} ^ {\star} - \widehat {V} _ {h + 1} \right] \leq 5 H ^ {2} \tag {101} \\ \end{array}
$$
for all  $1 \leq j \leq H$ . Here, the third line in (101) applies the following fact
$$
\begin{array}{l} \sum_ {h} \sum_ {s, a} d _ {h} ^ {\pi^ {*}} (s, a) \mathsf {V a r} _ {P _ {h} (\cdot | s, a)} \left(V _ {h + 1} ^ {\star}\right) \\ = \sum_ {h} \sum_ {s} d _ {h} ^ {\pi^ {\star}} (s, \pi^ {\star} (s)) \left[ \left\langle P _ {h} ^ {\pi^ {\star}} (s, \cdot), V _ {h + 1} ^ {\star} \circ V _ {h + 1} ^ {\star} \right\rangle - \left(\left\langle P _ {h} ^ {\pi^ {\star}} (s, \cdot), V _ {h + 1} ^ {\star} \right\rangle\right) ^ {2} \right] \\ = \sum_ {h} \sum_ {s} d _ {h} ^ {\pi^ {\star}} (s, \pi^ {\star} (s)) \left[ \left\langle P _ {h} ^ {\pi^ {\star}} (s, \cdot), V _ {h + 1} ^ {\star} \circ V _ {h + 1} ^ {\star} \right\rangle - \left(V _ {h} ^ {\star} (s) - r _ {h} ^ {\star} (s, \pi^ {\star} (s))\right) ^ {2} \right] \\ \leq \sum_ {h} \sum_ {s} d _ {h} ^ {\pi^ {\star}} (s, \pi^ {\star} (s)) \left[ \left\langle P _ {h} ^ {\pi^ {\star}} (s, \cdot), V _ {h + 1} ^ {\star} \circ V _ {h + 1} ^ {\star} \right\rangle - \left(V _ {h} ^ {\star} (s)\right) ^ {2} + 2 H \right] \\ = \sum_ {h} \left(d _ {h} ^ {\pi^ {*}}\right) ^ {\top} P _ {h} ^ {\pi^ {*}} \left(V _ {h + 1} ^ {*} \circ V _ {h + 1} ^ {*}\right) - \sum_ {h} \left(d _ {h} ^ {\pi^ {*}}\right) ^ {\top} \left(V _ {h} ^ {*} \circ V _ {h} ^ {*}\right) \bigg ] + 2 H ^ {2} \\ = \sum_ {h} \left(d _ {h + 1} ^ {\pi^ {\star}}\right) ^ {\top} \left(V _ {h + 1} ^ {\star} \circ V _ {h + 1} ^ {\star}\right) - \sum_ {h} \left(d _ {h} ^ {\pi^ {\star}}\right) ^ {\top} \left(V _ {h} ^ {\star} \circ V _ {h} ^ {\star}\right) + 2 H ^ {2} \\ \leq 2 H ^ {2}, \\ \end{array}
$$
where the second identity comes from the Bellman equation; the third relation uses the fact that  $V_{h}^{\star}(s) \leq H$ , and the penultimate line holds since  $\left(d_h^{\pi^*}\right)^\top P_h^{\pi^*} = \left(d_{h + 1}^{\pi^*}\right)^\top$ ; the penultimate step in (101) is due to (100); and the line in (101) holds true since  $\left(d_h^{\pi^*}\right)^\top P_h^{\pi^*} = \left(d_{h + 1}^{\pi^*}\right)^\top$ .