SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \left\langle Q _ {h} ^ {k _ {i}} \left(x _ {h}, \cdot\right), \pi_ {h} ^ {*} (\cdot | x _ {h}) - \pi_ {h} ^ {k _ {i}} (\cdot | x _ {h}) \right\rangle \\ \leq \frac {\alpha H ^ {2}}{2} + \frac {\operatorname {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \left\| \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})\right) - \operatorname {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \right. \right\| \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}))}{\alpha}. \\ \end{array}
$$
Proof. By the updating rule in (3.2), we have
$$
\pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) = \frac {\pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h}) \cdot \exp \left\{\alpha Q _ {h} ^ {k _ {i}} \left(x _ {h} , \cdot\right) \right\}}{\sum_ {a \in \mathcal {A}} \pi_ {h} ^ {k _ {i}} (a \mid x _ {h}) \cdot \exp \left\{\alpha Q _ {h} ^ {k _ {i}} \left(x _ {h} , a\right) \right\}}. \tag {B.5}
$$
For ease of presentation, we denote  $\Upsilon = \sum_{a\in \mathcal{A}}\pi_h^{k_i}(a\mid x_h)\cdot \exp \{\alpha Q_h^{k_i}(x_h,a)\}$ . Then we have
$$
\begin{array}{l} \left\langle \alpha Q _ {h} ^ {k _ {i}} \left(x _ {h}, \cdot\right), \pi_ {h} ^ {*} (\cdot | x _ {h}) - \pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) \right\rangle \\ = \left\langle \log \Upsilon + \log \pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) - \log \pi_ {h} ^ {k _ {i}} (\cdot | x _ {h}), \pi_ {h} ^ {*} (\cdot | x _ {h}) - \pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) \right\rangle \\ = \left\langle \log \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) - \log \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h}), \pi_ {h} ^ {*} (\cdot \mid x _ {h}) - \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) \right\rangle \tag {B.6} \\ \end{array}
$$
where the first equality uses (B.5), and the second equality follows from the fact that  $\sum_{a\in \mathcal{A}}\pi_h^* (a\mid x_h) - \pi_h^{k_{i + 1}}(a\mid x_h) = 0$ . Rearranging (B.6) gives that
$$
\begin{array}{l} \left. \left(\mathrm {B}. 6\right) = \left\langle \log \left(\frac {\pi_ {h} ^ {*} (\cdot \mid x _ {h})}{\pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})}\right), \pi_ {h} ^ {*} (\cdot \mid x _ {h}) \right\rangle - \left\langle \log \left(\frac {\pi_ {h} ^ {*} (\cdot \mid x _ {h})}{\pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h})}\right), \pi_ {h} ^ {*} (\cdot \mid x _ {h}) \right\rangle \right. \\ - \left\langle \log \left(\frac {\pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h})}{\pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})}\right), \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) \right\rangle \\ = \mathrm {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \| \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})\right) - \mathrm {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \| \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h})\right) \\ - \operatorname {K L} \left(\pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) \| \pi_ {h} ^ {k _ {i}} (\cdot | x _ {h})\right). \tag {B.7} \\ \end{array}
$$
$$
\begin{array}{l} \left\langle \alpha Q _ {h} ^ {k _ {i}} \left(x _ {h}, \cdot\right), \pi_ {h} ^ {*} (\cdot \mid x _ {h}) - \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h}) \right\rangle \\ = \left\langle \alpha Q _ {h} ^ {k _ {i}} \left(x _ {h}, \cdot\right), \pi_ {h} ^ {*} (\cdot | x _ {h}) - \pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) \right\rangle - \left\langle \alpha Q _ {h} ^ {k _ {i}} \left(x _ {h}, \cdot\right), \pi_ {h} ^ {k _ {i}} (\cdot | x _ {h}) - \pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) \right\rangle \\ \leq \mathrm {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \| \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})\right) - \mathrm {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \| \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h})\right) \\ - \mathrm {K L} \left(\pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) \| \pi_ {h} ^ {k _ {i}} (\cdot | x _ {h})\right) + \alpha H \cdot \| \pi_ {h} ^ {k _ {i}} (\cdot | x _ {h}) - \pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) \| _ {1}, \tag {B.8} \\ \end{array}
$$
where the last inequality uses (B.7), Cauchy-Schwarz inequality, and the fact that  $\| Q_h^{k_i}\|_\infty \leq H$ . By Pinsker's inequality, we have  $\mathrm{KL}(\pi_h^{k_{i + 1}}(\cdot |x_h)\| \pi_h^{k_i}(\cdot |x_h))\geq \| \pi_h^{k_{i + 1}}(\cdot |x_h) - \pi_h^{k_i}(\cdot |x_h)\| _1^2 /2$ . Together with (B.8), we obtain that
$$
\begin{array}{l} \left\langle \alpha Q _ {h} ^ {k _ {i}} \left(x _ {h}, \cdot\right), \pi_ {h} ^ {*} (\cdot \mid x _ {h}) - \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h}) \right\rangle \\ \leq \operatorname {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \| \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})\right) - \operatorname {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \| \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h})\right) \\ - \left\| \pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) - \pi_ {h} ^ {k _ {i}} (\cdot | x _ {h}) \right\| _ {1} ^ {2} / 2 + \alpha H \cdot \left\| \pi_ {h} ^ {k _ {i}} (\cdot | x _ {h}) - \pi_ {h} ^ {k _ {i + 1}} (\cdot | x _ {h}) \right\| _ {1} \\ \leq \mathrm {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \| \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})\right) - \mathrm {K L} \left(\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \| \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h})\right) + \alpha^ {2} H ^ {2} / 2, \tag {B.9} \\ \end{array}
$$
where the last inequality uses the fact that  $\max_{y\in \mathbb{R}}\{-y^2 /2 + \alpha H\cdot y\} = \alpha^2 H^2 /2$ . Rearranging (B.9) concludes the proof of Lemma B.2.
By Lemma B.2, we further have
$$
\begin{array}{l} \left. \right.\left(\mathrm {B}. 4\right) \leq B \sum_ {i = 1} ^ {L} \left( \right.\frac {\alpha H ^ {2}}{2} + \frac {\mathbb {E} _ {\pi^ {*}} \left[ \right. \mathrm {K L} \left( \right.\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \left. \right\| \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})\left. \right) - \mathrm {K L} \left( \right.\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \left. \right\| \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h})) \left. \right]\left. \right.}{\alpha}\left. \right) \\ = B \cdot \left( \right.\frac {\alpha H ^ {2} K}{2 B} + \frac {\mathbb {E} _ {\pi^ {*}} \left[ \right. \mathrm {K L} \left( \right.\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \left\| \right. \pi_ {h} ^ {k _ {1}} (\cdot \mid x _ {h})\left. \right) - \mathrm {K L} \left( \right.\pi_ {h} ^ {*} (\cdot \mid x _ {h}) \left\| \right. \pi_ {h} ^ {k _ {L + 1}} (\cdot \mid x _ {h})\left. \right)\left. \right]\left. \right.}{\alpha}\left. \right) \\ \leq \frac {\alpha H ^ {2} K}{2} + \frac {B \cdot \log | \mathcal {A} |}{\alpha}, \tag {B.10} \\ \end{array}
$$
where the equality uses the fact that  $L = K / B$ , and the last inequality uses the non-negativity of KL-divergence and the fact that  $\pi_h^{k_1} = \pi_h^1$  is the uniform policy. Combining (B.4), (B.10), and  $\alpha = \sqrt{2B\log|\mathcal{A}| / (KH^2)}$ , we obtain for all  $h\in [H]$ :
$$
\sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {*}} \left[ \left\langle Q _ {h} ^ {k} \left(x _ {h}, \cdot\right), \pi_ {h} ^ {*} (\cdot \mid x _ {h}) - \pi_ {h} ^ {k} (\cdot \mid x _ {h}) \right\rangle \right] \leq \sqrt {2 B H ^ {2} K \cdot \log | \mathcal {A} |}. \tag {B.11}
$$
Telescoping (B.11) across  $h \in [H]$  concludes the proof of Lemma 4.2.
B.3 Proof of Lemma 4.3
Proof. Recall that  $B \in \mathbb{N}^{+}$  is the batch size,  $k_{i} = (i - 1) \cdot B + 1$  and  $L = K / B$ . Fix  $h \in [H]$ . We have
$$
\begin{array}{l} \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {*}} [ \delta_ {h} ^ {k} (x _ {h}, a _ {h}) ] = \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {*}} \left[ r _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) + \mathbb {P} _ {h} V _ {h + 1} ^ {k} \left(x _ {h}, a _ {h}\right) - Q _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] \\ = \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {*}} \left[ r _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) + \mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}} \left(x _ {h}, a _ {h}\right) - Q _ {h} ^ {t _ {k}} \left(x _ {h}, a _ {h}\right) \right] \tag {B.12} \\ = \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {*}} \left[ r _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) - \bar {r} _ {h} ^ {t _ {k}} \left(x _ {h}, a _ {h}\right) \right] + \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {*}} \left[ \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}} - \widehat {\mathbb {P}} _ {h} ^ {t _ {k}} V _ {h + 1} ^ {t _ {k}}\right) \left(x _ {h}, a _ {h}\right) \right], \\ \end{array}
$$
where the first equality uses the definition of  $t_k$  and the updating rule, and the second equality uses the definition of  $Q_h^k = \bar{r}_h^k + \widehat{\mathbb{P}}_h^k V_{h+1}^k$  in (3.4). Furthermore, we have
$$
\begin{array}{l} \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {*}} \left[ r _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) - \bar {r} _ {h} ^ {t _ {k}} \left(x _ {h}, a _ {h}\right) \right] = \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {*}} \left[ r _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] - B \sum_ {i = 1} ^ {L} \mathbb {E} _ {\pi^ {*}} \left[ \bar {r} _ {h} ^ {k _ {i}} \left(x _ {h}, a _ {h}\right) \right] \\ = \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {*}} \left[ r _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] - \sum_ {i = 1} ^ {L} \sum_ {k = k _ {i} - 1} ^ {k _ {i} - 1} \mathbb {E} _ {\pi^ {*}} \left[ r _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] \\ = \sum_ {k = K - B + 1} ^ {K} \mathbb {E} _ {\pi^ {*}} \left[ r _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] \leq B, \tag {B.13} \\ \end{array}
$$
where the first equality uses the updating rule, the second equality follows from the definition of  $\bar{r}_h^k$  in (3.5), and the last inequality is obtained by the fact that  $\| r_h^k\|_\infty \leq 1$  for any  $(k,h)\in [K]\times [H]$ . Combining (B.12) and (B.13) and then taking summation across  $h\in [H]$ , we obtain that
$$
\sum_ {k = 1} ^ {K} \sum_ {h = 1} ^ {H} \mathbb {E} _ {\pi^ {*}} \left[ \delta_ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] \leq \sum_ {k = 1} ^ {K} \sum_ {h = 1} ^ {H} \mathbb {E} _ {\pi^ {*}} \left[ \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}} - \widehat {\mathbb {P}} _ {h} ^ {t _ {k}} V _ {h + 1} ^ {t _ {k}}\right) \left(x _ {h}, a _ {h}\right) \right] + B H. \tag {B.14}
$$
On the other hand, similar to the derivation of (B.12), we have
$$
\begin{array}{l} - \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {k}} \left[ \delta_ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] \\ = \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {t _ {k}}} \left[ \bar {r} _ {h} ^ {t _ {k}} \left(x _ {h}, a _ {h}\right) - r _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) \right] + \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {k}} \left[ \left(\widehat {\mathbb {P}} _ {h} ^ {t _ {k}} V _ {h + 1} ^ {t _ {k}} - \mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}\right) \left(x _ {h}, a _ {h}\right) \right]. \tag {B.15} \\ \end{array}
$$
For the first term of (B.15), by the updating rule and calculation, we have
$$
\begin{array}{l} \sum_ {k = 1} ^ {K} \mathbb {E} _ {\pi^ {t _ {k}}} [ \bar {r} _ {h} ^ {t _ {k}} (x _ {h}, a _ {h}) - r _ {h} ^ {k} (s _ {h}, a _ {h}) ] = B \sum_ {i = 1} ^ {L} \mathbb {E} _ {\pi^ {k _ {i}}} [ \bar {r} _ {h} ^ {k _ {i}} (x _ {h}, a _ {h}) ] - \sum_ {i = 1} ^ {L} \sum_ {k = k _ {i}} ^ {k _ {i + 1} - 1} \mathbb {E} _ {\pi^ {k _ {i}}} [ r _ {h} ^ {k} (x _ {h}, a _ {h}) ] \\ = \sum_ {i = 1} ^ {L} \sum_ {k = k _ {i - 1}} ^ {k _ {i} - 1} \mathbb {E} _ {\pi^ {k _ {i}}} \left[ r _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] - \sum_ {i = 1} ^ {L} \sum_ {k = k _ {i}} ^ {k _ {i + 1} - 1} \mathbb {E} _ {\pi^ {k _ {i}}} \left[ r _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] \\ \leq \sum_ {i = 1} ^ {L - 1} \sum_ {k = k _ {i}} ^ {k _ {i + 1} - 1} \left(\mathbb {E} _ {\pi^ {k _ {i + 1}}} \left[ r _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right] - \mathbb {E} _ {\pi^ {k _ {i}}} \left[ r _ {h} ^ {k} \left(x _ {h}, a _ {h}\right) \right]\right), \tag {B.16} \\ \end{array}
$$
where the last inequality uses the fact that  $-\sum_{k = k_L}^{k_{L + 1} - 1}\mathbb{E}_{\pi^{k_L}}[r_h^k (x_h,a_h)]\leq 0$ . Summing over  $h\in [H]$  in (B.16) gives that
$$
\begin{array}{l} \sum_ {k = 1} ^ {K} \sum_ {h = 1} ^ {H} \mathbb {E} _ {\pi^ {t _ {k}}} \left[ \bar {r} _ {h} ^ {t _ {k}} \left(x _ {h}, a _ {h}\right) - r _ {h} ^ {k} \left(s _ {h}, a _ {h}\right) \right] \\ \leq \sum_ {i = 1} ^ {L - 1} \sum_ {k = k _ {i}} ^ {k _ {i + 1} - 1} \left(V _ {1} ^ {\pi^ {k _ {i + 1}, k}} (x _ {1}) - V _ {1} ^ {\pi^ {k _ {i}}, k} (x _ {1})\right) \\ = \sum_ {i = 1} ^ {L - 1} \sum_ {k = k _ {i}} ^ {k _ {i + 1} - 1} \sum_ {h = 1} ^ {H} \mathbb {E} _ {\pi^ {k _ {i + 1}}} \left[ \left\langle Q _ {h} ^ {\pi^ {k _ {i}}, k} (\cdot \mid x _ {h}), \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) - \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h}) \right\rangle \right], \tag {B.17} \\ \end{array}
$$
where the last inequality uses the value difference lemma (Lemma B.1). By the policy updating rule in (3.2), we have
$$
\pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) = \frac {\pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h}) \cdot \exp \left(\alpha Q _ {h} ^ {k _ {i}} (x _ {h} , \cdot)\right)}{\sum_ {a \in \mathcal {A}} \pi_ {h} ^ {k _ {i}} (a \mid x _ {h}) \cdot \exp \left(\alpha Q _ {h} ^ {k _ {i}} (x _ {h} , a)\right)}.
$$
for any  $x_{h}\in S$ , which implies that
$$
\begin{array}{l} \frac {\pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})}{\pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h})} = \frac {\sum_ {a \in \mathcal {A}} \pi_ {h} ^ {k _ {i}} (a \mid x _ {h}) \cdot \exp (\alpha Q _ {h} ^ {k _ {i}} (x _ {h} , a))}{\exp (\alpha Q _ {h} ^ {k _ {i}} (x _ {h} , \cdot))} \\ \geq \frac {\sum_ {a \in \mathcal {A}} \pi_ {h} ^ {k _ {i}} (a \mid x _ {h})}{\exp (\alpha H)} = \exp (- \alpha H) \geq 1 - \alpha H, \\ \end{array}
$$
where the first inequality follows the fact that  $0 \leq Q_h^{k_i}(\cdot, \cdot) \leq H$ , and the last inequality uses the basic inequality  $\exp(y) \geq 1 + y$  for all  $y \in \mathbb{R}$ . Together with
$$
\pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) - \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h}) = \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) \cdot \left(1 - \frac {\pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h})}{\pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h})}\right),
$$
we further obtain
$$
\pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) - \pi_ {h} ^ {k _ {i}} (\cdot \mid x _ {h}) \leq \alpha H \cdot \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) \tag {B.18}
$$
for any  $x_{h}\in S$ . Plugging (B.18) into (B.17) gives that
$$
\leq \alpha H \sum_ {i = 1} ^ {L - 1} \sum_ {k = k _ {i}} ^ {k _ {i + 1} - 1} \sum_ {h = 1} ^ {H} \mathbb {E} _ {\pi^ {k _ {i + 1}}} \left[ \left\langle Q _ {h} ^ {\pi^ {k _ {i}}, k} \left(x _ {h}, \cdot\right), \pi_ {h} ^ {k _ {i + 1}} (\cdot \mid x _ {h}) \right\rangle \right] \leq \alpha H ^ {3} K, \tag {B.19}
$$
where the last inequality follows from the fact that  $Q_h^{\pi^{k_i}, k}(\cdot, \cdot) \leq H$ . Plugging (B.17) and (B.19) into (B.15), we have
$$
\begin{array}{l} - \sum_ {k = 1} ^ {K} \sum_ {h = 1} ^ {H} \mathbb {E} _ {\pi^ {k}} [ \delta_ {h} ^ {k} (x _ {h}, a _ {h}) ] \leq \sum_ {k = 1} ^ {K} \sum_ {h = 1} ^ {H} \mathbb {E} _ {\pi^ {k}} [ (\widehat {\mathbb {P}} _ {h} ^ {t _ {k}} V _ {h + 1} ^ {t _ {k}} - \mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}) (x _ {h}, a _ {h}) ] + \alpha H ^ {3} K \\ \leq \sum_ {k = 1} ^ {K} \sum_ {h = 1} ^ {H} \left(\widehat {\mathbb {P}} _ {h} ^ {t _ {k}} V _ {h + 1} ^ {t _ {k}} - \mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}\right) \left(x _ {h} ^ {k}, a _ {h} ^ {k}\right) + \alpha H ^ {3} K + \sqrt {H ^ {3} K \iota}, \tag {B.20} \\ \end{array}
$$
where the last inequality uses Azuma-Hoeffding inequality. Putting (B.14) and (B.20) together, we conclude the proof of Lemma 4.3.
B.4 Proof of Lemma 4.4
Proof. Recall that
$$
\left(\widehat {\mathbb {P}} _ {h} ^ {t _ {k}} V _ {h + 1} ^ {t _ {k}}\right) (\cdot , \cdot) = \min  \left\{\phi (\cdot , \cdot) ^ {\top} w _ {h} ^ {t _ {k}} + \Gamma_ {h} ^ {t _ {k}} (\cdot , \cdot), H - h \right\} ^ {+}, \tag {B.21}
$$
where  $w_{h}^{t_{k}}$  and  $\Gamma_h^{t_k}$  take form
$$
w _ {h} ^ {t _ {k}} = \left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1} \left(\sum_ {\tau = 1} ^ {t _ {k} - 1} \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) \cdot V _ {h + 1} ^ {t _ {k}} \left(x _ {h + 1} ^ {\tau}\right)\right), \quad \Gamma_ {h} ^ {t _ {k}} (\cdot , \cdot) = \beta \cdot \sqrt {\phi (\cdot , \cdot) ^ {\top} \left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1} \phi (\cdot , \cdot)}. \tag {B.22}
$$
Here  $\Lambda_h^{t_k}$  is the covariance matrix:
$$
\Lambda_ {h} ^ {t _ {k}} = \sum_ {\tau = 1} ^ {t _ {k} - 1} \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) ^ {\top} + \lambda \cdot I _ {d}. \tag {B.23}
$$
Back to our proof, for any  $(x,a)\in S\times \mathcal{A}$  , we have
$$
\begin{array}{l} (\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}) (x, a) = \phi (x, a) ^ {\top} \langle \mu_ {h}, V _ {h + 1} ^ {t _ {k}} \rangle_ {\mathcal {S}} \\ = \phi (x, a) ^ {\top} (\Lambda_ {h} ^ {t _ {k}}) ^ {- 1} \Lambda_ {h} ^ {t _ {k}} \langle \mu_ {h}, V _ {h + 1} ^ {t _ {k}} \rangle_ {\mathcal {S}} \\ = \phi (x, a) ^ {\top} \left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1} \Big (\sum_ {\tau = 1} ^ {t _ {k} - 1} \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) ^ {\top} \langle \mu_ {h}, V _ {h + 1} ^ {t _ {k}} \rangle_ {\mathcal {S}} + \lambda \cdot \langle \mu_ {h}, V _ {h + 1} ^ {t _ {k}} \rangle_ {\mathcal {S}} \Big) \\ = \phi (x, a) ^ {\top} \left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1} \Big (\sum_ {\tau = 1} ^ {t _ {k} - 1} \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) \cdot \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}\right) \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) + \lambda \cdot \langle \mu_ {h}, V _ {h + 1} ^ {t _ {k}} \rangle_ {\mathcal {S}} \Big), \tag {B.24} \\ \end{array}
$$
where the first and the last equality follows from the definition of linear MDP (Definition 2.1), and the third equality uses (B.23). Putting (B.22) and (B.24) together, we have
$$
\begin{array}{l} \phi (x, a) ^ {\top} w _ {h} ^ {t _ {k}} - \mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}} (x, a) \\ = \underbrace {\phi \left(x , a\right) ^ {\top} \left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1} \left(\sum_ {\tau = 1} ^ {t _ {k} - 1} \phi \left(x _ {h} ^ {\tau} , a _ {h} ^ {\tau}\right) \cdot \left(V _ {h + 1} ^ {t _ {k}} \left(x _ {h + 1} ^ {\tau}\right) - \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}\right) \left(x _ {h} ^ {\tau} , a _ {h} ^ {\tau}\right)\right)\right)} _ {(\star)} \tag {B.25} \\ - \underbrace {\lambda \cdot \phi (x , a) ^ {\top} (\Lambda_ {h} ^ {k}) ^ {- 1} \langle \mu_ {h} , V _ {h + 1} ^ {t _ {k}} \rangle_ {\mathcal {S}}} _ {(\star \star)}. \\ \end{array}
$$
For Term  $(\star)$  in (B.25), by Cauchy-Schwarz inequality, we have
$$
\begin{array}{l} (\star) \leq \sqrt {\phi (x , a) ^ {\top} \left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1} \phi (x , a)} \cdot \left\| \sum_ {\tau = 1} ^ {t _ {k} - 1} \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) \cdot \left(V _ {h + 1} ^ {t _ {k}} \left(x _ {h + 1} ^ {\tau}\right) - \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}\right) \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right)\right) \right\| _ {\left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1}} \\ \leq \mathcal {O} \left(d ^ {1 / 4} H K ^ {1 / 4} \iota^ {1 / 2}\right) \cdot \sqrt {\phi (x , a) ^ {\top} \left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1} \phi (x , a)}, \tag {B.26} \\ \end{array}
$$
where the last inequality follows from the following lemma.
Lemma B.3. Fix  $\delta \in (0,1]$ . It holds for all  $(k,h) \in [K] \times [H]$  that
$$
\left\| \sum_ {\tau = 1} ^ {t _ {k} - 1} \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) \cdot \left(V _ {h + 1} ^ {t _ {k}} \left(x _ {h + 1} ^ {\tau}\right) - \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}\right) \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right)\right) \right\| _ {\left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1}} \leq \mathcal {O} \left(d ^ {1 / 4} H K ^ {1 / 4} \iota^ {1 / 2}\right).
$$
Proof. See §C for a detailed proof.
For Term  $(\star \star)$  in (B.25), we have
$$
\begin{array}{l} (\star \star) \leq \lambda \cdot \sqrt {\phi (x , a) ^ {\top} \left(\Lambda_ {h} ^ {k}\right) ^ {- 1} \phi (x , a)} \cdot \left\| \langle \mu_ {h}, V _ {h + 1} ^ {t _ {k}} \rangle_ {\mathcal {S}} \right\| _ {\left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1}} \\ \leq \sqrt {\lambda} \cdot \sqrt {\phi (x , a) ^ {\top} \left(\Lambda_ {h} ^ {k}\right) ^ {- 1} \phi (x , a)} \cdot \left\| \langle \mu_ {h}, V _ {h + 1} ^ {t _ {k}} \rangle_ {\mathcal {S}} \right\| _ {2} \\ \leq \sqrt {\lambda d H ^ {2}} \cdot \sqrt {\phi (x , a) ^ {\top} \left(\Lambda_ {h} ^ {k}\right) ^ {- 1} \phi (x , a)}, \tag {B.27} \\ \end{array}
$$
where the second inequality uses the fact that  $\lambda \cdot I_d \preceq \Lambda_h^{t_k}$ , and last inequality follows from  $\| \langle \mu_h, V_{h+1}^{t_k} \rangle_S \|_2 \leq H\sqrt{d}$ , which is implied by Definition 2.1. Plugging (B.26) and (B.27) into (B.25), together with  $\lambda = 1$ , we obtain
$$
| \phi (x, a) ^ {\top} w _ {h} ^ {t _ {k}} - \mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}} (x, a) | \leq \beta \sqrt {\phi (x , a) ^ {\top} \left(\Lambda_ {h} ^ {t _ {k}}\right) ^ {- 1} \phi (x , a)} = \Gamma_ {h} ^ {t _ {k}} (x, a) \tag {B.28}
$$
for any  $(x,a)\in S\times \mathcal{A}$  with  $\beta = \mathcal{O}(d^{1 / 4}HK^{1 / 4}\iota^{1 / 2})$ . Together with the definition of  $\widehat{\mathbb{P}}_h^{t_k}V_{h + 1}^{t_k}$  in (B.21), we have
$$
\begin{array}{l} \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}} - \widehat {\mathbb {P}} _ {h} ^ {t _ {k}} V _ {h + 1} ^ {t _ {k}}\right) (x, a) \\ \leq \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}\right) (x, a) - \min  \left\{\phi (x, a) ^ {\top} w _ {h} ^ {t _ {k}} + \Gamma_ {h} ^ {t _ {k}} (x, a), H - h \right\} \\ = \max \{(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}) (x, a) - \phi (x, a) ^ {\top} w _ {h} ^ {t _ {k}} - \Gamma_ {h} ^ {t _ {k}} (x, a), (\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}) (x, a) - (H - h) \}, \\ \leq \max  \{0, 0 \} = 0, \tag {B.29} \\ \end{array}
$$
where the last inequality uses (B.28) and the fact that  $\| V_{h + 1}^{t_k}\|_{\infty}\leq H - h$ . Moreover, by (B.28), we have
$$
\begin{array}{l} \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}} - \widehat {\mathbb {P}} _ {h} ^ {t _ {k}} V _ {h + 1} ^ {t _ {k}}\right) (x, a) \geq \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}}\right) (x, a) - \phi (x, a) ^ {\top} w _ {h} ^ {t _ {k}} - \Gamma_ {h} ^ {t _ {k}} (x, a) \\ \geq - 2 \Gamma_ {h} ^ {t _ {k}} (x, a). \tag {B.30} \\ \end{array}
$$
Combining (B.29), (B.30), and the fact that  $(\mathbb{P}_hV_{h + 1}^{t_k} - \widehat{\mathbb{P}}_h^{t_k}V_{h + 1}^{t_k})(\cdot ,\cdot)\geq -2H$  we obtain
$$
- 2 \min  \{H, \Gamma_ {h} ^ {t _ {k}} (x, a) \} \leq \left(\mathbb {P} _ {h} V _ {h + 1} ^ {t _ {k}} - \widehat {\mathbb {P}} _ {h} ^ {t _ {k}} V _ {h + 1} ^ {t _ {k}}\right) (x, a) \leq 0,
$$
which concludes the proof of Lemma 4.4.
B.5 Proof of Lemma 4.5
Proof. For ease of presentation, we define
$$
\Gamma_ {h} ^ {k} (\cdot , \cdot) = \beta \sqrt {\phi (\cdot , \cdot) ^ {\top} \left(\Lambda_ {h} ^ {k}\right) ^ {- 1} \phi (\cdot , \cdot)}, \quad \Lambda_ {h} ^ {k} = \sum_ {\tau = 1} ^ {k - 1} \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) \phi \left(x _ {h} ^ {\tau}, a _ {h} ^ {\tau}\right) ^ {\top} + \lambda I _ {d}, \tag {B.31}
$$
for all  $(k,h)\in [K]\times [H]$ . Then we have the following lemma, which uses the elliptical potential to bound  $\sum_{k = 1}^{K}\sum_{h = 1}^{H}\Gamma_h^k (x_h^k,a_h^k)$
Lemma B.4. It holds that
$$
\sum_ {k = 1} ^ {K} \sum_ {h = 1} ^ {H} \Gamma_ {h} ^ {k} \left(x _ {h} ^ {k}, a _ {h} ^ {k}\right) \leq \mathcal {O} \left(d ^ {3 / 4} H ^ {2} K ^ {3 / 4} \cdot \iota\right).
$$
Proof. By the definition of  $\Gamma_h^k$  in (B.31), we have
$$
\sum_ {k = 1} ^ {K} \sum_ {h = 1} ^ {H} \Gamma_ {h} ^ {k} \left(x _ {h} ^ {k}, a _ {h} ^ {k}\right) = \beta \sum_ {k = 1} ^ {K} \sum_ {h = 1} ^ {H} \sqrt {\phi \left(x _ {h} ^ {k} , a _ {h} ^ {k}\right) ^ {\top} \left(\Lambda_ {h} ^ {k}\right) ^ {- 1} \phi \left(x _ {h} ^ {k} , a _ {h} ^ {k}\right)}. \tag {B.32}
$$
Applying Cauchy-Schwarz inequality to (B.32), we have