SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathsf {o} - \mathsf {d e c} _ {\gamma} ^ {D} (\mathcal {M}) = \sup  _ {\mu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \sup  _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \left[ f ^ {\overline {{M}}} \left(\pi_ {\overline {{M}}}\right) - f ^ {M} (\pi) - \gamma \cdot D ^ {\pi} \left(\overline {{M}} \| M\right) \right] \\ \leq \sup  _ {\mu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \sup  _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\bar {M} \sim \mu} \left[ f ^ {\bar {M}} \left(\pi_ {\bar {M}}\right) - f ^ {\bar {M}} (\pi) - \frac {\gamma}{2} \cdot D ^ {\pi} \left(\bar {M} \| M\right) \right] + \frac {L _ {\mathrm {l i p}} ^ {2}}{2 \gamma}. \tag {33} \\ \end{array}
$$
Consider any fixed choice for  $\mu \in \Delta (\mathcal{M})$ . By Sion's minimax theorem (see Foster et al. [12] for details), we have
$$
\begin{array}{l} \inf  _ {p \in \Delta (\Pi)} \sup  _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \left[ f ^ {\overline {{M}}} \left(\pi_ {\overline {{M}}}\right) - f ^ {\overline {{M}}} (\pi) - \frac {\gamma}{2} \cdot D ^ {\pi} \left(\overline {{M}} \| M\right) \right] \\ = \inf  _ {p \in \Delta (\Pi)} \sup  _ {\nu \in \Delta (\mathcal {M})} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \mathbb {E} _ {M \sim \nu} \left[ f ^ {\overline {{M}}} (\pi_ {\overline {{M}}}) - f ^ {\overline {{M}}} (\pi) - \frac {\gamma}{2} \cdot D ^ {\pi} \big (\overline {{M}} \| M \big) \right] \\ = \sup _ {\nu \in \Delta (\mathcal {M})} \inf _ {p \in \Delta (\Pi)} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \mathbb {E} _ {M \sim \nu} \Big [ f ^ {\overline {{M}}} (\pi_ {\overline {{M}}}) - f ^ {\overline {{M}}} (\pi) - \frac {\gamma}{2} \cdot D ^ {\pi} \big (\overline {{M}} \| M \big) \Big ], \\ \end{array}
$$
so that the main term in (33) is equal to
$$
\begin{array}{l} \sup_{\mu \in \Delta (\mathcal{M})}\sup_{\nu \in \Delta (\mathcal{M})}\inf_{p\in \Delta (\Pi)}\mathbb{E}_{\pi \sim p}\mathbb{E}_{\overline{M}\sim \mu}\mathbb{E}_{M\sim \nu}\Big[f^{\overline{M}}(\pi_{\overline{M}}) - f^{\overline{M}}(\pi) - \frac{\gamma}{2}\cdot D^{\pi}\big(\overline{M}\parallel M\big)\Big] \\ \leq \sup  _ {\nu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \sup  _ {\mu \in \Delta (\mathcal {M})} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \mathbb {E} _ {M \sim \nu} \left[ f ^ {\overline {{M}}} (\pi_ {\overline {{M}}}) - f ^ {\overline {{M}}} (\pi) - \frac {\gamma}{2} \cdot D ^ {\pi} \left(\overline {{M}} \| M\right) \right] \\ = \sup  _ {\nu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \sup  _ {\overline {{M}} \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {M \sim \nu} \Big [ f ^ {\overline {{M}}} (\pi_ {\overline {{M}}}) - f ^ {\overline {{M}}} (\pi) - \frac {\gamma}{2} \cdot D ^ {\pi} \big (\overline {{M}} \| M \big) \Big ]. \\ \end{array}
$$
Relabeling, this is equal to
$$
\sup  _ {\mu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \sup  _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \left[ f ^ {M} \left(\pi_ {M}\right) - f ^ {M} (\pi) - \frac {\gamma}{2} \cdot D ^ {\pi} \left(M \| \overline {{M}}\right) \right] = \sup  _ {\mu \in \Delta (\mathcal {M})} \underline {{\operatorname* {d e c}}} _ {\gamma / 2} ^ {\check {D}} (\mathcal {M}, \mu).
$$
We now prove the lower bound. Using Assumption 3.1 and the AM-GM inequality once more, we have
$$
\begin{array}{l} \mathsf {o} - \mathsf {d e c} _ {\gamma} ^ {D} (\mathcal {M}) = \sup  _ {\mu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \sup  _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \left[ f ^ {\overline {{M}}} \left(\pi_ {\overline {{M}}}\right) - f ^ {M} (\pi) - \gamma \cdot D ^ {\pi} \left(\overline {{M}} \| M\right) \right] \\ \geq \sup  _ {\mu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \sup  _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \left[ f ^ {\overline {{M}}} \left(\pi_ {\overline {{M}}}\right) - f ^ {\overline {{M}}} (\pi) - \frac {3 \gamma}{2} \cdot D ^ {\pi} \left(\overline {{M}} \| M\right) \right] - \frac {L _ {\mathrm {l i p}} ^ {2}}{2 \gamma}. \\ \geq \sup  _ {\mu \in \Delta (\mathcal {M})} \sup  _ {\nu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \mathbb {E} _ {M \sim \nu} \left[ f ^ {\bar {M}} \left(\pi_ {\bar {M}}\right) - f ^ {\bar {M}} (\pi) - \frac {3 \gamma}{2} \cdot D ^ {\pi} \left(\bar {M} \| M\right) \right] - \frac {L _ {\mathrm {l i p}} ^ {2}}{2 \gamma}. \tag {34} \\ \end{array}
$$
Using the minimax theorem in the same fashion as before, the main term in (34) is equal to
$$
\begin{array}{l} \sup  _ {\nu \in \Delta (\mathcal {M})} \sup  _ {\mu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \mathbb {E} _ {M \sim \nu} \left[ f ^ {\overline {{M}}} (\pi_ {\overline {{M}}}) - f ^ {\overline {{M}}} (\pi) - \frac {3 \gamma}{2} \cdot D ^ {\pi} \left(\overline {{M}} \| M\right) \right] \\ = \sup _ {\nu \in \Delta (\mathcal {M})} \inf _ {p \in \Delta (\Pi)} \sup _ {\mu \in \Delta (\mathcal {M})} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \mathbb {E} _ {M \sim \nu} \left[ f ^ {\overline {{M}}} (\pi_ {\overline {{M}}}) - f ^ {\overline {{M}}} (\pi) - \frac {3 \gamma}{2} \cdot D ^ {\pi} \big (\overline {{M}} \| M \big) \right] \\ = \sup  _ {\nu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \sup  _ {\overline {{M}} \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {M \sim \nu} \left[ f ^ {\overline {{M}}} (\pi_ {\overline {{M}}}) - f ^ {\overline {{M}}} (\pi) - \frac {3 \gamma}{2} \cdot D ^ {\pi} \big (\overline {{M}} \| M \big) \right]. \\ \end{array}
$$
$$
\sup _ {\mu \in \Delta (\mathcal {M})} \inf _ {p \in \Delta (\Pi)} \sup _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \left[ f ^ {M} (\pi_ {M}) - f ^ {M} (\pi) - \frac {3 \gamma}{2} \cdot D ^ {\pi} (M \| \overline {{M}}) \right] = \sup _ {\mu \in \Delta (\mathcal {M})} \underline {{\operatorname* {d e c}}} _ {3 \gamma / 2} ^ {\check {D}} (\mathcal {M}, \mu).
$$
Proof of Proposition 3.2. Let  $\overline{M}$  be arbitrary. By Sion's minimax theorem (see Foster et al. [12] for details), we have
$$
\mathsf {d e c} _ {\gamma} ^ {D} (\mathcal {M}, \overline {{M}}) = \sup  _ {\mu \in \Delta (\mathcal {M})} \inf  _ {p \in \Delta (\Pi)} \mathbb {E} _ {\pi \sim p, M \sim \mu} \left[ f ^ {M} \left(\pi_ {M}\right) - f ^ {M} (\pi) - \gamma \cdot D ^ {\pi} \left(\overline {{M}} \| M\right) \right].
$$
By the assumed triangle inequality for  $D$ , we have that for all  $\pi \in \Pi$
$$
\begin{array}{l} \mathbb {E} _ {M, M ^ {\prime} \sim \mu} \left[ D ^ {\pi} (M \| M ^ {\prime}) \right] \leq C \mathbb {E} _ {M \sim \mu} \left[ D ^ {\pi} (\bar {M} \| M) \right] + C \mathbb {E} _ {M ^ {\prime} \sim \mu} \left[ D ^ {\pi} (\bar {M} \| M ^ {\prime}) \right] \\ = 2 C \mathbb {E} _ {M \sim \mu} \left[ D ^ {\pi} \left(\bar {M} \parallel M\right) \right]. \\ \end{array}
$$
Applying this bound above, we have that
$$
\begin{array}{l} \mathsf{dec}_{\gamma}^{D}(\mathcal{M},\overline{M})\leq \sup_{\mu \in \Delta (\mathcal{M})}\inf_{p\in \Delta (\Pi)}\mathbb{E}_{\pi \sim p,M\sim \mu}\Bigl[f^{M}(\pi_{M}) - f^{M}(\pi) - \frac{\gamma}{2C}\cdot \mathbb{E}_{M^{\prime}\sim \mu}[D^{\pi}(M^{\prime}\parallel M)]\Bigr] \\ \leq \sup _ {\nu \in \Delta (\mathcal {M})} \sup _ {\mu \in \Delta (\mathcal {M})} \inf _ {p \in \Delta (\Pi)} \mathbb {E} _ {\pi \sim p, M \sim \mu} \left[ f ^ {M} (\pi_ {M}) - f ^ {M} (\pi) - \frac {\gamma}{2 C} \cdot \mathbb {E} _ {M ^ {\prime} \sim \nu} [ D ^ {\pi} (M ^ {\prime} \| M) ] \right] \\ \leq \sup _ {\nu \in \Delta (\mathcal {M})} \inf _ {p \in \Delta (\Pi)} \sup _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \left[ f ^ {M} \left(\pi_ {M}\right) - f ^ {M} (\pi) - \frac {\gamma}{2 C} \cdot \mathbb {E} _ {M ^ {\prime} \sim \nu} [ D ^ {\pi} (M ^ {\prime} \| M) ] \right] \\ = \sup  _ {\nu \in \Delta (\mathcal {M})} \underline {{\operatorname {d e c}}} _ {\gamma / (2 C)} ^ {D} (\mathcal {M}, \nu). \\ \end{array}
$$
Proof of Proposition 3.3. The upper bound on  $\mathsf{o - dec}_{\gamma}^{\mathrm{bi}}(\mathcal{M})$  follows from Proposition 2.2, so it remains to prove the lower bound on  $\mathsf{dec}_{\gamma}^{\mathrm{bi}}(\mathcal{M})$ .
For a model  $M$ , define the Bellman operator  $\mathcal{T}_h^M$  via
$$
\left[ \mathcal {T} _ {h} ^ {M} Q \right] (s, a) = \mathbb {E} _ {s _ {h + 1} \sim P _ {h} ^ {M} (\cdot | s, a), r _ {h} \sim R _ {h} ^ {M} (\cdot | s, a)} \left[ r _ {h} + \max  _ {a ^ {\prime} \in \mathcal {A}} Q \left(s _ {h + 1}, a ^ {\prime}\right) \right]. \tag {35}
$$
$$
D _ {\mathbf {s b e}} ^ {\pi} (Q \parallel M) := \sum_ {h = 1} ^ {H} \mathbb {E} ^ {M, \pi} \left[ \left(Q _ {h} \left(s _ {h}, a _ {h}\right) - \left[ \mathcal {T} _ {h} ^ {M} Q _ {h + 1} \right] \left(s _ {h}, a _ {h}\right)\right) ^ {2} \right]
$$
and  $D_{\mathrm{sbe}}^{\pi}(\overline{M}\parallel M)\coloneqq D_{\mathrm{sbe}}^{\pi}(Q^{\overline{M},\star}\parallel M)$ . By Jensen's inequality, it suffices to lower bound  $\operatorname{dec}_{\gamma}^{D_{\mathrm{sbe}}}(\mathcal{M})$ .
Let  $\mathcal{S} = \{\mathfrak{s},\mathfrak{t}\}$  and  $\mathcal{A} = \{\mathfrak{a},\mathfrak{b}\}$ . We consider a sub-family  $\mathcal{M}' \subset \mathcal{M}$  of deterministic combination lock MDPs parameterized by  $\vec{a} = (\vec{a}_1,\dots,\vec{a}_H) \in \mathcal{A}^H$ , with  $M_{\vec{a}}$  defined as follows.
- The initial state is  $s_1 = \mathfrak{s}$ .
- For each  $h = 1, \dots, H - 1$ , if  $s_h = \mathfrak{s}$ , then selecting  $a_h = \vec{a}_h$  transitions to  $s_{h + 1} = \mathfrak{s}$ , and selecting  $a_h \neq \vec{a}_h$  transitions to  $s_{h + 1} = \mathfrak{t}$ .  $\mathfrak{t}$  is a self-looping terminal state: if  $s_h = \mathfrak{t}$ , then  $s_{h + 1} = \mathfrak{t}$  regardless of the action taken.
- If  $s_H = \mathfrak{s}$  and  $a_H = \vec{a}_H$ , then  $r_H = \Delta > 0$ ; all other state-action tuples have zero reward.
We choose  $\overline{M}$  such that  $Q_h^{\overline{M},\star}(s,a) = 0$  for all  $(h,s,a)$ . We calculate that for all  $\vec{a} \in \mathcal{A}^H$ , and all policies  $\pi \in \Pi_{\mathrm{RNS}}$ ,
-  $f^{M_{\vec{a}}}(\pi_{M_{\vec{a}}}) - f^{M_{\vec{a}}}(\pi) = \Delta \cdot \mathbb{P}^{M_{\vec{a}},\pi}(a_{1:H}\neq \vec{a}).$
-  $D_{\mathrm{she}}^{\pi}\left(Q^{M,\star} \parallel M_{\vec{a}}\right) = \Delta^{2} \cdot \mathbb{P}^{M_{\vec{a}},\pi}(a_{1:H} = \vec{a})$
$$
\begin{array}{l} \mathsf {d e c} _ {\gamma} ^ {D _ {\mathrm {s b e}}} (\mathcal {M}, \overline {{M}}) \geq \inf  _ {p \in \Delta (\Pi)} \max  _ {\vec {a} \in \mathcal {A} ^ {H}} \mathbb {E} _ {\pi \sim p} \left[ \Delta \cdot \mathbb {P} ^ {M _ {\vec {a}}, \pi} (a _ {1: H} \neq \vec {a}) - \gamma \cdot \Delta^ {2} \cdot \mathbb {P} ^ {M _ {\vec {a}}, \pi} (a _ {1: H} = \vec {a}) \right] \\ \geq \inf _ {p \in \Delta (\Pi)} \mathbb {E} _ {\vec {a} \sim \mathrm {U n i f} (\mathcal {A} ^ {H})} \mathbb {E} _ {\pi \sim p} \big [ \Delta \cdot \mathbb {P} ^ {M _ {\vec {a}}, \pi} (a _ {1: H} \neq \vec {a}) - \gamma \cdot \Delta^ {2} \cdot \mathbb {P} ^ {M _ {\vec {a}}, \pi} (a _ {1: H} = \vec {a}) \big ]. \\ \end{array}
$$
It is straightforward to see by induction that for all  $\pi \in \Pi_{\mathrm{RNS}}$ ,  $\mathbb{E}_{\vec{a}\sim \mathrm{Unif}(\mathcal{A}^H)}[\mathbb{P}^{M_{\vec{a}},\pi}(a_{1:H} = \vec{a})]\leq 2^{-H}$  and  $\mathbb{E}_{\vec{a}\sim \mathrm{Unif}(\mathcal{A}^H)}[\mathbb{P}^{M_{\vec{a}},\pi}(a_{1:H}\neq \vec{a})]\geq \frac{1}{2}$ , so we have
$$
\operatorname {d e c} _ {\gamma} ^ {D _ {\text {s b e}}} (\mathcal {M}, \overline {{M}}) \geq \frac {\Delta}{2} - \gamma \frac {\Delta^ {2}}{2 ^ {H}}.
$$
The result follows by choosing  $\Delta$  appropriately.
Proof of Proposition 3.4. Let  $H$  be given, and assume without loss of generality that  $S = 2^{H - 1} + 2^{H - 2}$ . Let  $N := 2^{H - 2}$ . We construct a family of MDPs  $\mathcal{M} = \{M_1, \dots, M_N\}$  with deterministic rewards and transitions as follows.
-  $\mathcal{A} = \{\mathfrak{a},\mathfrak{b}\}$
- The state space is  $\mathcal{S} = \mathcal{T} \cup \mathcal{E} \cup \{\mathfrak{s}\}$ .  $\mathfrak{s}$  is a deterministic initial state which appears only in layer 1.  $\mathcal{T}$  represents a depth  $H - 2$  binary tree (with the root at layer  $h = 2$ ), which has  $N = 2^{H - 2}$  leaf states in layer  $H$ , labeled by  $\{1, \dots, N\}$ , and  $2^{H - 1} - 1$  states in total.  $\mathcal{E} = \{N + 1, \dots, 2N\}$  is an auxiliary collection of self-looping terminal states, each of which can appear in layers  $h = 2, \dots, H$ . In total, we have  $|S| = 2^{H - 1} + 2^{H - 2}$ .
The dynamics and rewards for MDP  $M_{i}$  are as follows.
- At layer  $h = 1$ ,  $s_1 = \mathfrak{s}$  deterministically.
- If  $a_1 = \mathfrak{a}$ , we transition to  $s_2 = N + i \in \mathcal{E}$ , and if  $a_1 = \mathfrak{b}$ , we transition to the root state for the tree  $\mathcal{T}$ . We receive no reward.
- For  $h \geq 2$ , all states  $s \in \mathcal{E}$  are self-looping and have no reward (i.e.  $s_{h+1} = s_h$  if  $s_h \in \mathcal{E}$  for  $h > 1$ ).
- For  $h \geq 2$ , states in  $\mathcal{T}$  follow a standard deterministic binary tree structure (e.g., [24, 8]). Beginning from the root node at  $h = 2$ , action  $\mathfrak{a}$  transitions to the left successor, while action  $\mathfrak{b}$  transitions to the right successor. For  $M_{i}$ , we receive reward 1 for reaching the leaf node  $i \in \mathcal{T}$  at layer  $H$ , and receive zero reward for all other states. Note that the transition probabilities for this portion of the MDP do not depend on  $i$ .
Online estimation. We first construct an online estimation algorithm for the class  $\mathcal{M}$ . Recall that we adopt Hellinger distance, given by  $D_{\mathsf{H}}^{\pi} \left( \widehat{M}, M \right) = D_{\mathsf{H}}^{2} \left( \widehat{M}(\pi), M(\pi) \right)$ .
We first note that since all  $M \in \mathcal{M}$  have  $f^{M}(\pi_{M}) = 1$ , the optimistic estimation error is equal to the (non-optimistic) estimation error
$$
\mathbf {E s t} ^ {\mathsf {H}} = \sum_ {t = 1} ^ {T} \mathbb {E} _ {\pi^ {t} \sim p ^ {t}} \mathbb {E} _ {\widehat {M} ^ {t} \sim \mu^ {t}} \left[ D _ {\mathsf {H}} ^ {2} \left(\widehat {M} ^ {t} \left(\pi^ {t}\right), M ^ {\star} \left(\pi^ {t}\right)\right) \right].
$$
Observe that since all MDPs  $M \in \mathcal{M}$  have deterministic rewards and transitions, there exists a function  $o^M(\pi)$  such that  $o \sim M(\pi)$  has  $o = o^M(\pi)$  almost surely. It follows that
$$
D _ {\mathsf {H}} ^ {2} (M (\pi), M ^ {\prime} (\pi)) = 2 \mathbb {I} \{o ^ {M} (\pi) \neq o ^ {M ^ {\prime}} (\pi) \}
$$
for all  $M, M' \in \mathcal{M}$ , so we have
$$
\mathbf {E s t} ^ {\mathsf {H}} = \sum_ {t = 1} ^ {T} \mathbb {E} _ {\pi^ {t} \sim p ^ {t}} \mathbb {E} _ {\widehat {M} ^ {t} \sim \mu^ {t}} \left[ D _ {\mathsf {H}} ^ {2} \Big (\widehat {M} ^ {t} (\pi^ {t}), M ^ {\star} (\pi^ {t}) \Big) \right] = 2 \sum_ {t = 1} ^ {T} \mathbb {E} _ {\pi^ {t} \sim p ^ {t}} \mathbb {E} _ {\widehat {M} ^ {t} \sim \mu^ {t}} \left[ \mathbb {I} \{o ^ {\widehat {M} ^ {t}} (\pi^ {t}) \neq o ^ {M ^ {\star}} (\pi^ {t}) \} \right].
$$
For the estimation algorithm, we choose
$$
\mu^ {t} (M) \propto \exp \left(- \sum_ {i <   t} \mathbb {I} \{o ^ {M} (\pi^ {i}) \neq o ^ {i} \}\right).
$$
Lemma C.3 implies that with probability 1, the sequence  $\pi^1,\dots ,\pi^T$  satisfies
$$
\sum_ {t = 1} ^ {T} \mathbb {E} _ {\widehat {M} ^ {t} \sim \mu^ {t}} \left[ \mathbb {I} \{o ^ {\widehat {M} ^ {t}} (\pi^ {t}) \neq o ^ {t} \} \right] - \sum_ {t = 1} ^ {T} \mathbb {I} \{o ^ {M ^ {*}} (\pi^ {t}) \neq o ^ {t} \} \leq \frac {1}{2} \sum_ {t = 1} ^ {T} \mathbb {E} _ {\widehat {M} ^ {t} \sim \mu^ {t}} \left[ \mathbb {I} \{o ^ {\widehat {M} ^ {t}} (\pi^ {t}) \neq o ^ {t} \} \right] + \log | \mathcal {M} |.
$$
Since  $o^{M^{\star}}(\pi^{t}) = o^{t}$ , rearranging yields
$$
\sum_ {t = 1} ^ {T} \mathbb {E} _ {\widehat {M} ^ {t} \sim \mu^ {t}} \left[ \mathbb {I} \{o ^ {\widehat {M} ^ {t}} (\pi^ {t}) \neq o ^ {t} \} \right] \leq 2 \log | \mathcal {M} |.
$$
From here, a standard application of Freedman's inequality [16] implies that with probability at least  $1 - \delta$ ,  $\sum_{t=1}^{T} \mathbb{E}_{\pi^t \sim p^t} \mathbb{E}_{\widehat{M}^t \sim \mu^t} \left[ \mathbb{I} \{ o^{\widehat{M}^t}(\pi^t) \neq o^t \} \right] \lesssim \log (|\mathcal{M}| / \delta) \lesssim \log (S / \delta)$ .
Lower bound for posterior sampling. Observe that for all  $M_i \in \mathcal{M}$ , the unique optimal policy has  $\pi_{M_i}(\mathfrak{s}) = \mathfrak{b}$  at  $h = 1$ . Thus, the posterior sampling algorithm, which chooses  $p^t(\pi) = \mu^t(\{M \in \mathcal{M} \mid \pi_M = \pi\})$ , will never play  $a_1 = \mathfrak{a}$ , and will never encounter states in  $\mathcal{E}$ . As a result, the problem is equivalent (for this algorithm) to a multi-armed bandit problem with  $N$  arms and noiseless binary rewards, which requires  $\mathbb{E}[\mathrm{Reg}_{\mathrm{DM}}] \gtrsim N \gtrsim S$  in the worst-case [21].
Upper bound for Algorithm 2. We first bound the Optimistic DEC for  $\mathcal{M}$ . For any  $\mu \in \Delta(\mathcal{M})$ , we can write
$$
\begin{array}{l} \mathbf{o - d e c} _ {\gamma} ^ {\mathrm {H}} (\mathcal {M}, \mu) = \inf  _ {p \in \Delta (\Pi)} \sup  _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \left[ f ^ {\overline {{M}}} \left(\pi_ {\overline {{M}}}\right) - f ^ {M} (\pi) - \gamma \cdot D \left(\overline {{M}} (\pi) \| M (\pi)\right) \right] \\ = \inf  _ {p \in \Delta (\Pi)} \sup  _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \left[ f ^ {\overline {{M}}} \left(\pi_ {\overline {{M}}}\right) - f ^ {M} (\pi) - 2 \gamma \cdot \mathbb {I} \left\{o ^ {\overline {{M}}} (\pi) \neq o ^ {M} (\pi) \right\} \right] \\ = \inf _ {p \in \Delta (\Pi)} \sup _ {M \in \mathcal {M}} \mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \left[ f ^ {M} (\pi_ {M}) - f ^ {M} (\pi) - 2 \gamma \cdot \mathbb {I} \{o ^ {\overline {{M}}} (\pi) \neq o ^ {M} (\pi) \} \right], \\ \end{array}
$$
where the last equality uses that  $f^M(\pi_M) = 1$  for all  $M \in \mathcal{M}$ . We choose  $p = (1 - \varepsilon)\mu(\{M \in \mathcal{M} \mid \pi_M = \cdot\}) + \varepsilon\pi_{\mathfrak{a}}$ , where  $\pi_{\mathfrak{a}}$  is the policy that plays action  $\mathfrak{a}$  deterministically.
Now, let  $M \in \mathcal{M}$  be fixed. Since each model  $M_{i}$  transitions to  $s_2 = N + i$  deterministically when  $a_1 = \mathfrak{a}$ , we
$$
\mathbb {E} _ {\pi \sim p} \mathbb {I} \left\{o ^ {\overline {{M}}} (\pi) \neq o ^ {M} (\pi) \right\} \geq \varepsilon \cdot \mathbb {I} \left\{o ^ {\overline {{M}}} \left(\pi_ {\mathfrak {a}}\right) \neq o ^ {M} \left(\pi_ {\mathfrak {a}}\right) \right\} = \varepsilon \cdot \mathbb {I} \left\{M \neq \overline {{M}} \right\}
$$
$$
\mathbb {E} _ {\overline {{M}} \sim \mu} \mathbb {E} _ {\pi \sim p} \mathbb {I} \{o ^ {\overline {{M}}} (\pi) \neq o ^ {M} (\pi) \} \geq \varepsilon \mu (\mathcal {M} \setminus \{M \}).
$$
$$
\mathbb {E} _ {\pi \sim p} \left[ f ^ {M} \left(\pi_ {M}\right) - f ^ {M} (\pi) \right] = \varepsilon + (1 - \varepsilon) \mu \left(\mathcal {M} \setminus \{M \}\right).
$$
By choosing  $\varepsilon = \gamma^{-1}$ , which is admissible whenever  $\gamma \geq 1$ , we have
$$
\mathbb {E} _ {\pi \sim p} \mathbb {E} _ {\overline {{M}} \sim \mu} \big [ f ^ {M} (\pi_ {M}) - f ^ {M} (\pi) - 2 \gamma \cdot \mathbb {I} \{o ^ {\overline {{M}}} (\pi) \neq o ^ {M} (\pi) \} \big ] \leq \varepsilon + \mu (\mathcal {M} \setminus \{M \}) - 2 \gamma \varepsilon \mu (\mathcal {M} \setminus \{M \}) \leq \frac {1}{\gamma}.
$$
This establishes that
$$
\mathsf {o} \text {- d e c} _ {\gamma} ^ {\mathsf {H}} (\mathcal {M}) \leq \frac {1}{\gamma}
$$
for all  $\gamma \geq 1$ . A regret bound of the form  $\mathbb{E}[\mathrm{Reg}_{\mathrm{DM}}] \leq \tilde{O}(\sqrt{T \log(S)})$  now follows by invoking Theorem 2.1 with the estimation guarantee in the prequel and choosing  $\gamma$  appropriately.