SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathbb {E} \left[ \sum_ {h = 1} ^ {H} \sum_ {(s, a) \in \mathcal {S} \times \mathcal {A}} \rho (s) \left(\pi_ {h} ^ {\operatorname {m i x}} (a | s) - \pi_ {h} ^ {\mathrm {N B C U}} (a | s)\right) r _ {h} (s, a) \mathbb {I} \{n _ {h} ^ {\mathrm {U}} (s) = 0 \} \right] \\ = \sum_ {h = 1} ^ {H} \sum_ {(s, a) \in \mathcal {S} \times \mathcal {A}} \rho (s) r _ {h} (s, a) \mathbb {E} \left[ \left(\pi_ {h} ^ {\operatorname {m i x}} (a | s) - \pi_ {h} ^ {\mathrm {N B C U}} (a | s)\right) \mathbb {I} \{n _ {h} ^ {\mathrm {U}} (s) = 0 \} \right] \\ = \sum_ {h = 1} ^ {H} \sum_ {(s, a) \in \mathcal {S} \times \mathcal {A}} \rho (s) r _ {h} (s, a) \mathbb {P} \left(n _ {h} ^ {\mathrm {U}} (s) = 0\right) \mathbb {E} \left[ \pi_ {h} ^ {\operatorname {m i x}} (a | s) - \pi_ {h} ^ {\mathrm {N B C U}} (a | s) \mid n _ {h} ^ {\mathrm {U}} (s) = 0 \right] \\ = \sum_ {h = 1} ^ {H} \sum_ {(s, a) \in \mathcal {S} \times \mathcal {A}} \rho (s) r _ {h} (s, a) \mathbb {P} \left(n _ {h} ^ {\mathrm {U}} (s) = 0\right) \left(\pi_ {h} ^ {\operatorname {m i x}} (a | s) - \frac {1}{| \mathcal {A} |}\right) \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(a)} {=} \sum_ {h = 1} ^ {H} \sum_ {s \in \mathcal {S}} \rho (s) \mathbb {P} \left(n _ {h} ^ {\mathrm {U}} (s) = 0\right) \left(\eta - \frac {1}{| \mathcal {A} |}\right) \\ \stackrel {(b)} {=} H \left(\eta - \frac {1}{| \mathcal {A} |}\right) \sum_ {s \in \mathcal {S}} \rho (s) \mathbb {P} \left(n _ {1} ^ {\mathrm {U}} (s) = 0\right). \\ \end{array}
$$
In the equation (a), we use the fact that  $r_h(s, a^1) = 1$  but  $r_h(s, a) = 0$  for any  $a \neq a^1$ . In the equation (b), since each state is an absorbing state, we have that  $\mathbb{P}(n_h^{\mathrm{U}}(s) = 0) = \mathbb{P}(n_1^{\mathrm{U}}(s) = 0)$  for any  $h \in [H]$ . We consider two cases to address RHS of equation (b). In the first case of  $\eta \geq 1 / |\mathcal{A}|$ , we directly have that
$$
\mathbb {E} \left[ \sum_ {h = 1} ^ {H} \sum_ {(s, a) \in \mathcal {S} \times \mathcal {A}} \rho (s) \left(\pi_ {h} ^ {\mathrm {m i x}} (a | s) - \pi_ {h} ^ {\mathrm {N B C U}} (a | s)\right) r _ {h} (s, a) \mathbb {I} \{n _ {h} ^ {\mathrm {U}} (s) = 0 \} \right] \geq 0.
$$
By Eq. (15), we have that
$$
\mathbb {E} \left[ V (\pi^ {\mathrm {m i x}}) - V (\pi^ {\mathrm {N B C U}}) \right] \geq 0,
$$
$$
\mathbb {E} \left[ V (\pi^ {\mathrm {E}}) - V \left(\pi^ {\mathrm {N B C U}}\right) \right] \geq (1 - \eta) \left(V \left(\pi^ {\mathrm {E}}\right) - V \left(\pi^ {\beta}\right)\right).
$$
In the second case of  $\eta < 1 / |\mathcal{A}|$ , we have that
$$
\begin{array}{l} H \left(\eta - \frac {1}{| \mathcal {A} |}\right) \sum_ {s \in \mathcal {S}} \rho (s) \mathbb {P} \left(n _ {1} ^ {\mathrm {U}} (s) = 0\right) \overset {(a)} {\geq} - \left(\frac {1}{| \mathcal {A} |} - \eta\right) H \exp \left(- \frac {N _ {\mathrm {t o t}}}{| \mathcal {S} |}\right) \\ \geq - (1 - \eta) H \exp \left(- \frac {N _ {\mathrm {t o t}}}{| \mathcal {S} |}\right) \\ \stackrel {(b)} {\geq} - \frac {(1 - \eta) H}{2}. \\ \end{array}
$$
In the inequality  $(a)$ , we use that
$$
\sum_ {s \in \mathcal {S}} \rho (s) \mathbb {P} \left(n _ {1} ^ {\mathrm {U}} (s) = 0\right) = \sum_ {s \in \mathcal {S}} \rho (s) (1 - \rho (s)) ^ {N _ {\mathrm {t o t}}} = \left(1 - \frac {1}{| \mathcal {S} |}\right) ^ {N _ {\mathrm {t o t}}} \leq \exp \left(- \frac {N _ {\mathrm {t o t}}}{| \mathcal {S} |}\right).
$$
The inequality  $(b)$  holds since we consider the range where  $N_{\mathrm{tot}} \geq |\mathcal{S}| \log(2)$ . By Eq. (15), we have that
$$
\mathbb {E} \left[ V (\pi^ {\mathrm {m i x}}) - V (\pi^ {\mathrm {N B C U}}) \right] \geq - \frac {(1 - \eta) H}{2}.
$$
$$
\begin{array}{l} \mathbb {E} \left[ V (\pi^ {\mathrm {E}}) - V (\pi^ {\mathrm {N B C U}}) \right] \geq (1 - \eta) \left(V (\pi^ {\mathrm {E}}) - V (\pi^ {\beta})\right) - \frac {(1 - \eta) H}{2} \\ = \frac {(1 - \eta)}{2} \left(V \left(\pi^ {\mathrm {E}}\right) - V \left(\pi^ {\beta}\right)\right). \\ \end{array}
$$
In both cases, we prove that  $\mathbb{E}\left[V(\pi^{\mathrm{E}}) - V(\pi^{\mathrm{NBCU}})\right] \gtrsim (1 - \eta)(V(\pi^{\mathrm{E}}) - V(\pi^{\beta}))$  and thus complete the proof.
D Proof of Results in Section 5
D.1 Proof of Proposition 2
In the tabular case, with the first-order optimality condition, we have  $c_h^\star(s,a) = \widehat{d_h^{\mathrm{E}}}(s,a) / (\widehat{d_h^{\mathrm{E}}}(s,a) + \widehat{d_h^{\mathrm{U}}}(s,a))$ . By Eq. (5), we have
$$
\widehat {d _ {h} ^ {\mathrm {U}}} (s, a) w _ {h} (s, a) = \widehat {d _ {h} ^ {\mathrm {U}}} (s, a) \times \frac {\widehat {d _ {h} ^ {\mathrm {E}}} (s , a)}{\widehat {d _ {h} ^ {\mathrm {U}}} (s , a)} = \widehat {d _ {h} ^ {\mathrm {E}}} (s, a).
$$
Hence, the learning objective (3) reduces to (1).
$$
\Delta_{h}(\theta) = \min_{(s,a)\in \mathcal{D}_{h}^{\mathrm{E}}\cup \mathcal{D}_{h}^{\mathrm{S},1}}\langle \theta ,\phi_{h}(s,a)\rangle -\max_{(s^{\prime},a^{\prime})\in \mathcal{D}_{h}^{\mathrm{S},2}}\langle \theta ,\phi_{h}(s^{\prime},a^{\prime})\rangle .
$$
$$
\begin{array}{l} \Delta_ {h} (\bar {\theta} _ {h}) - \Delta_ {h} (\theta) = \min  _ {(s, a) \in \mathcal {D} _ {h} ^ {\mathrm {E} \cup} \mathcal {D} _ {h} ^ {\mathrm {S}, 1}} \langle \bar {\theta} _ {h}, \phi_ {h} (s, a) \rangle - \max  _ {(s ^ {\prime}, a ^ {\prime}) \in \mathcal {D} _ {h} ^ {\mathrm {S}, 2}} \langle \bar {\theta} _ {h}, \phi_ {h} (s ^ {\prime}, a ^ {\prime}) \rangle \\ - \min  _ {(s, a) \in \mathcal {D} _ {h} ^ {\mathrm {E}} \cup \mathcal {D} _ {h} ^ {\mathrm {S}, 1}} \langle \theta , \phi_ {h} (s, a) \rangle + \max  _ {(s ^ {\prime}, a ^ {\prime}) \in \mathcal {D} _ {h} ^ {\mathrm {S}, 2}} \langle \theta , \phi_ {h} (s ^ {\prime}, a ^ {\prime}) \rangle \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(a)} {\leq} \langle \bar {\theta} _ {h}, \phi_ {h} (s ^ {1}, a ^ {1}) \rangle - \langle \bar {\theta} _ {h}, \phi_ {h} (s ^ {2}, a ^ {2}) \rangle - \langle \theta , \phi_ {h} (s ^ {1}, a ^ {1}) \rangle + \langle \theta , \phi_ {h} (s ^ {2}, a ^ {2}) \rangle \\ = \left\langle \bar {\theta} _ {h} - \theta , \phi_ {h} \left(s ^ {1}, a ^ {1}\right) - \phi_ {h} \left(s ^ {2}, a ^ {2}\right) \right\rangle \\ \stackrel {(b)} {\leq} \left\| \bar {\theta} _ {h} - \theta \right\| \left\| \phi_ {h} (s ^ {1}, a ^ {1}) - \phi_ {h} (s ^ {2}, a ^ {2}) \right\|. \\ \end{array}
$$
In inequality  $(a)$ , we utilize the facts that  $(s^1, a^1) \in \mathrm{argmin}_{(s,a) \in \mathcal{D}_h^{\mathrm{E}} \cup \mathcal{D}_h^{\mathrm{S},1}} \langle \theta_h, \phi_h(s,a) \rangle$  and
$(s^2, a^2) \in \operatorname{argmax}_{(s,a) \in \mathcal{D}_h^{S,2}} \langle \theta_h, \phi_h(s,a) \rangle$ . Inequality (b) follows the Cauchy-Schwarz inequality.
Let  $L_{h} = \left\| \phi_{h}(s^{1},a^{1}) - \phi_{h}(s^{2},a^{2})\right\|$  and we finish the proof.
D.3 Proof of Lemma 2
First, by Taylor's Theorem, there exists  $\theta_h^\prime \in \{\theta \in \mathbb{R}^d:\theta^t = \theta_h^\star +t(\bar{\theta}_h - \theta_h^\star),\forall t\in [0,1]\}$  such that
$$
\begin{array}{l} \mathcal {L} _ {h} (\bar {\theta} _ {h}) = \mathcal {L} _ {h} \left(\theta_ {h} ^ {\star}\right) + \langle \nabla \mathcal {L} _ {h} \left(\theta_ {h} ^ {\star}\right), \bar {\theta} _ {h} - \theta_ {h} ^ {\star} \rangle + \frac {1}{2} \left(\bar {\theta} _ {h} - \theta_ {h} ^ {\star}\right) ^ {\top} \nabla^ {2} \mathcal {L} _ {h} \left(\theta_ {h} ^ {\prime}\right) \left(\bar {\theta} _ {h} - \theta_ {h} ^ {\star}\right) \\ = \mathcal {L} _ {h} \left(\theta_ {h} ^ {\star}\right) + \frac {1}{2} \left(\bar {\theta} _ {h} - \theta_ {h} ^ {\star}\right) ^ {\top} \nabla^ {2} \mathcal {L} _ {h} \left(\theta_ {h} ^ {\prime}\right) \left(\bar {\theta} _ {h} - \theta_ {h} ^ {\star}\right). \tag {16} \\ \end{array}
$$
The last equality follows the optimality condition that  $\nabla \mathcal{L}_h(\theta_h^\star) = 0$ . Then, our strategy is to prove that the smallest eigenvalue of the Hessian matrix  $\nabla^2\mathcal{L}_h(\theta_h')$  is positive, i.e.,  $\lambda_{\mathrm{min}}(\nabla^{2}\mathcal{L}_{h}(\theta_{h}^{\prime})) > 0$ . We first calculate the Hessian matrix  $\nabla^2\mathcal{L}_h(\theta_h')$ . Given  $\mathcal{D}^{\mathrm{E}}$  and  $\mathcal{D}^{\mathrm{U}}$ , we define the function  $G:\mathbb{R}^{(|\mathcal{D}^{\mathrm{E}}| + |\mathcal{D}^{\mathrm{U}}|)}\to \mathbb{R}$  as
$$
G (v) \triangleq \frac {1}{| \mathcal {D} ^ {\mathrm {E}} |} \sum_ {i = 1} ^ {| \mathcal {D} ^ {\mathrm {E}} |} g (v _ {i}) + \frac {1}{| \mathcal {D} ^ {\mathrm {U}} |} \sum_ {j = 1} ^ {| \mathcal {D} ^ {\mathrm {U}} |} g (v _ {j}),
$$
where  $v_{i}$  is the  $i$ -th element in the vector  $v \in \mathbb{R}^{(|\mathcal{D}^{\mathrm{E}}| + |\mathcal{D}^{\mathrm{U}}|)}$  and  $g(x) = \log (1 + \exp (x))$  is a real-valued function. Besides, we use  $B_{h} \in \mathbb{R}^{(|\mathcal{D}^{\mathrm{E}}| + |\mathcal{D}^{\mathrm{U}}|)\times d}$  to denote the matrix whose  $i$ -th row  $B_{h,i} = -y_i\phi_h(s^i,a^i)^\top$ , and  $y_{i} = 1$  if  $(s^i,a^i) \in \mathcal{D}_h^{\mathrm{E}}$ ,  $y_{i} = -1$  if  $(s^i,a^i) \notin \mathcal{D}_h^{\mathrm{E}}$ . Then the objective function can be reformulated as
$$
\begin{array}{l} \mathcal {L} _ {h} \left(\theta_ {h}\right) \\ = \sum_ {(s, a)} \widehat {d _ {h} ^ {\mathrm {E}}} (s, a) [ \log (1 + \exp (- \langle \phi_ {h} (s, a), \theta_ {h} \rangle)) ] + \sum_ {(s, a)} \widehat {d _ {h} ^ {\mathrm {U}}} (s, a) [ \log (1 + \exp (\langle \phi_ {h} (s, a), \theta_ {h} \rangle)) ] \\ = \frac {1}{\left| \mathcal {D} ^ {\mathrm {E}} \right|} \sum_ {(s, a) \in \mathcal {D} ^ {\mathrm {E}}} \log \left(1 + \exp \left(- \left\langle \phi_ {h} (s, a), \theta_ {h} \right\rangle\right)\right) + \frac {1}{\left| \mathcal {D} ^ {\mathrm {U}} \right|} \sum_ {(s, a) \in \mathcal {D} ^ {\mathrm {U}}} \log \left(1 + \exp \left(\left\langle \phi_ {h} (s, a), \theta_ {h} \right\rangle\right)\right) \\ = G \left(B _ {h} \theta_ {h}\right). \\ \end{array}
$$
Then we have that  $\nabla^2\mathcal{L}_h(\theta_h) = B_h^\top \nabla^2 G(B_h\theta_h)B_h$  , where
$$
\begin{array}{l} \nabla^ {2} G (B _ {h} \theta_ {h}) \\ = \operatorname {d i a g} \left(\frac {g ^ {\prime \prime} \left(\left(B _ {h} \theta_ {h}\right) _ {1}\right)}{\left| \mathcal {D} ^ {\mathrm {E}} \right|}, \dots , \frac {g ^ {\prime \prime} \left(\left(B _ {h} \theta_ {h}\right) _ {\left| \mathcal {D} ^ {\mathrm {E}} \right|}\right)}{\left| \mathcal {D} ^ {\mathrm {E}} \right|}, \frac {g ^ {\prime \prime} \left(\left(B _ {h} \theta_ {h}\right) _ {\left| \mathcal {D} ^ {\mathrm {E}} \right| + 1}\right)}{\left| \mathcal {D} ^ {\mathrm {E}} \right| + \left| \mathcal {D} ^ {\mathrm {U}} \right|}, \dots , \frac {g ^ {\prime \prime} \left(\left(B _ {h} \theta_ {h}\right) _ {\left| \mathcal {D} ^ {\mathrm {E}} \right| + \left| \mathcal {D} ^ {\mathrm {U}} \right|}\right)}{\left| \mathcal {D} ^ {\mathrm {E}} \right| + \left| \mathcal {D} ^ {\mathrm {U}} \right|}\right). \\ \end{array}
$$
Here  $g''(x) = \sigma(x)(1 - \sigma(x))$ , where  $\sigma(x) = 1 / (1 + \exp(-x))$  is the sigmoid function. The eigenvalues of  $\nabla^2 G(B_h\theta_h)$  are
$$
\left\{\frac {g ^ {\prime \prime} ((B _ {h} \theta_ {h}) _ {1})}{| \mathcal {D} ^ {\mathrm {E}} |}, \ldots , \frac {g ^ {\prime \prime} ((B _ {h} \theta_ {h}) _ {| \mathcal {D} ^ {\mathrm {E}} |})}{| \mathcal {D} ^ {\mathrm {E}} |}, \frac {g ^ {\prime \prime} ((B _ {h} \theta_ {h}) _ {| \mathcal {D} ^ {\mathrm {E}} | + 1})}{| \mathcal {D} ^ {\mathrm {E}} | + | \mathcal {D} ^ {\mathrm {U}} |}, \ldots , \frac {g ^ {\prime \prime} ((B _ {h} \theta_ {h}) _ {| \mathcal {D} ^ {\mathrm {E}} | + | \mathcal {D} ^ {\mathrm {U}} |})}{| \mathcal {D} ^ {\mathrm {E}} | + | \mathcal {D} ^ {\mathrm {U}} |} \right\}.
$$
Notice that  $\theta_h^\prime \in \{\theta \in \mathbb{R}^d:\theta^t = \theta_h^\star +t(\bar{\theta}_h - \theta_h^\star),\forall t\in [0,1]\}$ . For a matrix  $A$ , we use  $\lambda_{\mathrm{min}}(A)$  to denote the minimal eigenvalue of  $A$ . Here we claim that the minimum of the minimal eigenvalues of  $\nabla^2 G(B_h\theta^t)$  over  $t\in [0,1]$  is achieved at  $t = 0$  or  $t = 1$ . That is,
$$
\min \{\lambda_ {\min } (\nabla^ {2} G (B _ {h} \theta^ {t})): \forall t \in [ 0, 1 ] \} = \min \{\lambda_ {\min } (\nabla^ {2} G (B _ {h} \theta^ {0})), \lambda_ {\min } (\nabla^ {2} G (B _ {h} \theta^ {1})) \}.
$$
We prove this claim as follows. For any  $t \in [0,1]$ , we use  $\{\lambda_1(t), \dots, \lambda_{|\mathcal{D}^{\mathrm{E}}| + |\mathcal{D}^{\mathrm{U}}|}(t)\}$  to denote the eigenvalues of  $\nabla^2 G(B_h\theta^t)$ . For each  $i \in [| \mathcal{D}^{\mathrm{E}}| + |\mathcal{D}^{\mathrm{U}}|]$ , we consider  $\lambda_i(t): [0,1] \to \mathbb{R}$  as a function of  $t$ . Specifically,
$$
\lambda_ {i} (t) = \left\{ \begin{array}{l l} \frac {g ^ {\prime \prime} ((B _ {h} \theta_ {h} ^ {\star}) _ {i} + t (B _ {h} (\bar {\theta} _ {h} - \theta_ {h} ^ {\star})) _ {i})}{| \mathcal {D} ^ {\mathrm {E}} |}, & \quad \text {i f i \in [ | \mathcal {D} ^ {\mathrm {E}} | ]} \\ \frac {g ^ {\prime \prime} ((B _ {h} \theta_ {h} ^ {\star}) _ {i} + t (B _ {h} (\bar {\theta} _ {h} - \theta_ {h} ^ {\star})) _ {i})}{| \mathcal {D} ^ {\mathrm {E}} | + | \mathcal {D} ^ {\mathrm {U}} |}, & \quad \text {o t h e r w i s e .} \end{array} \right.
$$
We observe that  $g'''(x) = \sigma(x)(1 - \sigma(x))(1 - 2\sigma(x))$  which satisfies that  $\forall x \leq 0$ ,  $g'''(x) \geq 0$ , and  $\forall x \geq 0$ ,  $g'''(x) \leq 0$ . Therefore, we have that the minimum of  $\lambda_i(t)$  over  $t \in [0,1]$  must be achieved at  $t = 0$  or  $t = 1$ . That is,
$$
\min  _ {t \in [ 0, 1 ]} \lambda_ {i} (t) = \min  \left\{\lambda_ {i} (0), \lambda_ {i} (1) \right\}. \tag {17}
$$
For any  $t \in [0,1]$ , we define  $i^t \in [|D^{\mathrm{E}}| + |D^{\mathrm{U}}|]$  as the index of the minimal eigenvalue of  $\nabla^2 G(B_h\theta^t)$ , i.e.,  $\lambda_{i^t}(t) = \lambda_{\min}(\nabla^2 G(B_h\theta^t))$ . Then we have that
$$
\begin{array}{l} \min  \left\{ \right.\lambda_ {\min } \left( \right.\nabla^ {2} G \left(B _ {h} \theta^ {t}\right): \forall t \in [ 0, 1 ] \left. \right\} = \min  \left\{\lambda_ {i ^ {t}} (t): \forall t \in [ 0, 1 ] \right\} \\ \stackrel {(a)} {=} \min  \left\{\min  \left\{\lambda_ {i ^ {t}} (0), \lambda_ {i ^ {t}} (1) \right\}: \forall t \in [ 0, 1 ] \right\} \\ = \min  \left\{\lambda_ {i ^ {0}} (0), \lambda_ {i ^ {1}} (1) \right\} \\ \stackrel {(b)} {=} \min  \{\lambda_ {\min } (\nabla^ {2} G (B _ {h} \theta^ {0})), \lambda_ {\min } (\nabla^ {2} G (B _ {h} \theta^ {1})) \} \\ \end{array}
$$
Equality (a) follows (17) and equality (b) follows that  $\lambda_{i^0}(0)$  and  $\lambda_{i^1}(1)$  are the minimal eigenvalues of  $\nabla^2 G(B_h\theta^0)$  and  $\nabla^2 G(B_h\theta^1)$ , respectively.
In summary, we derive that
$$
\min  \left\{ \right.\lambda_ {\min } \left( \right.\nabla^ {2} G \left(B _ {h} \theta^ {t}\right): \forall t \in [ 0, 1 ] \left. \right\} = \min  \left\{ \right.\lambda_ {\min } \left( \right.\nabla^ {2} G \left(B _ {h} \theta^ {0}\right), \lambda_ {\min } \left(\nabla^ {2} G \left(B _ {h} \theta^ {1}\right)\right)\left. \right\}, \left. \right. \tag {18}
$$
which proves the previous claim.
Further, we consider  $\lambda_{\mathrm{min}}\left(\nabla^{2}\mathcal{L}_{h}(\theta_{h})\right)$ .
$$
\begin{array}{l} \lambda_ {\min } \left(\nabla^ {2} \mathcal {L} _ {h} (\theta_ {h})\right) = \inf  _ {x \in \mathbb {R} ^ {d}: \| x \| = 1} x ^ {\top} \nabla^ {2} \mathcal {L} _ {h} (\theta_ {h}) x \\ = \inf  _ {x \in \mathbb {R} ^ {d}: \| x \| = 1} \left(B _ {h} x\right) ^ {\top} \nabla^ {2} G \left(B _ {h} \theta_ {h}\right) \left(B _ {h} x\right) \\ = \inf  _ {z \in \operatorname {I m} \left(B _ {h}\right)} z ^ {\top} \nabla^ {2} G \left(B _ {h} \theta_ {h}\right) z \\ = \left(\inf  _ {z \in \operatorname {I m} \left(B _ {h}\right)} \| z \|\right) ^ {2} \lambda_ {\min } \left(\nabla^ {2} G \left(B _ {h} \theta_ {h}\right)\right) \\ \geq \left(\inf _ {z \in \operatorname {I m} (B _ {h})} \| z \|\right) ^ {2} \min  \{\lambda_ {\min } (\nabla^ {2} G (B _ {h} \theta^ {0})), \lambda_ {\min } (\nabla^ {2} G (B _ {h} \theta^ {1})) \}. \\ \end{array}
$$
Here  $\operatorname{Im}(B_h) = \{z \in \mathbb{R}^d : z = B_h x, \|x\| = 1\}$ . The last inequality follows Eq. (18).
Recall we assume that  $\mathbf{rank}(A_h) = d$ , so we have that  $\mathbf{rank}(B_h) = d$ . Thus,  $\operatorname{Im}(B_h)$  is a set of vectors with positive norms, i.e.,  $\inf_{z \in \operatorname{Im}(B_h)} \|z\| > 0$ . Besides, since  $g''(x) = \sigma(x)(1 - \sigma(x)) > 0$  we also have that
$$
\min  \left\{\lambda_ {\min } \left(\nabla^ {2} G \left(B _ {h} \theta^ {0}\right)\right), \lambda_ {\min } \left(\nabla^ {2} G \left(B _ {h} \theta^ {1}\right)\right) \right\} > 0.
$$
In summary, we obtain that
$$
\lambda_ {\min } \left(\nabla^ {2} \mathcal {L} _ {h} \left(\theta_ {h}\right)\right) \geq \left(\inf  _ {z \in \operatorname {I m} \left(B _ {h}\right)} \| z \|\right) ^ {2} \min  \left\{\lambda_ {\min } \left(\nabla^ {2} G \left(B _ {h} \theta^ {0}\right)\right), \lambda_ {\min } \left(\nabla^ {2} G \left(B _ {h} \theta^ {1}\right)\right) \right\} > 0.
$$
Then, with Eq. (16), there exists
$$
\tau_ {h} = \left(\inf  _ {z \in \operatorname {I m} \left(B _ {h}\right)} \| z \|\right) ^ {2} \min  \left\{\lambda_ {\min } \left(\nabla^ {2} G \left(B _ {h} \theta^ {0}\right)\right), \lambda_ {\min } \left(\nabla^ {2} G \left(B _ {h} \theta^ {1}\right)\right) \right\} > 0
$$
$$
\mathcal {L} _ {h} (\bar {\theta} _ {h}) \geq \mathcal {L} _ {h} \left(\theta_ {h} ^ {\star}\right) + \frac {\tau_ {h}}{2} \left\| \bar {\theta} _ {h} - \theta_ {h} ^ {\star} \right\| ^ {2},
$$
First, invoking Lemma 1 with  $\theta = \theta_h^\star$  yields that
$$
\Delta_ {h} \left(\theta_ {h} ^ {\star}\right) \geq \Delta_ {h} \left(\bar {\theta} _ {h}\right) - L _ {h} \left\| \bar {\theta} _ {h} - \theta_ {h} ^ {\star} \right\|.
$$
Here  $L_{h} = \|\phi_{h}(s,a) - \phi_{h}(s^{\prime},a^{\prime})\|$  with  $(s,a)\in \mathrm{argmin}_{(s,a)\in \mathcal{D}_h^{\mathrm{E}}\cup \mathcal{D}_h^{\mathrm{S},1}}\langle \theta_h^\star ,\phi_h(s,a)\rangle$  and  $(s^{\prime},a^{\prime})\in \mathrm{argmax}_{(s,a)\in \mathcal{D}_h^{\mathrm{S},2}}\langle \theta_h^\star ,\phi_h(s,a)\rangle$ . Then, by Lemma 2, there exists  $\tau_h > 0$  such that
$$
\mathcal {L} _ {h} (\bar {\theta} _ {h}) \geq \mathcal {L} _ {h} \left(\theta_ {h} ^ {\star}\right) + \frac {\tau_ {h}}{2} \left\| \bar {\theta} _ {h} - \theta_ {h} ^ {\star} \right\| ^ {2}.
$$
This directly implies an upper bound of the distance between  $\bar{\theta}_h$  and  $\theta_h^\star$ .
$$
\left\| \bar {\theta} _ {h} - \theta_ {h} ^ {\star} \right\| \leq \sqrt {\frac {2 \left(\mathcal {L} _ {h} (\bar {\theta} _ {h}) - \mathcal {L} _ {h} \left(\theta_ {h} ^ {\star}\right)\right)}{\tau_ {h}}}.
$$
If the feature is designed such that  $\sqrt{\frac{2\left(\mathcal{L}_h(\bar{\theta}_h) - \mathcal{L}_h(\theta_h^*)\right)}{\tau_h}} < \frac{\Delta_h(\bar{\theta}_h)}{L_h}$  holds, we further have that  $\|\bar{\theta}_h - \theta_h^*\| < \Delta_h(\bar{\theta}_h) / L_h$ . Then we get that
$$
\Delta_ {h} \left(\theta_ {h} ^ {\star}\right) \geq \Delta_ {h} \left(\bar {\theta} _ {h}\right) - L _ {h} \left\| \bar {\theta} _ {h} - \theta_ {h} ^ {\star} \right\| > 0,
$$
which completes the proof of the first statement.
Then we proceed to prove the imitation gap bound. We first identify the property of  $\pi^{\mathrm{ISW - BC}}$ . Recall the objective of WBCU.
$$
\pi^{\mathrm{ISW - BC}}\in \operatorname *{argmax}_{\pi}\sum_{h = 1}^{H}\sum_{(s,a)\in \mathcal{S}\times \mathcal{A}}\left\{\widehat{d_{h}^{\mathrm{U}}} (s,a)\times [w_{h}(s,a)\log \pi_{h}(a|s)]\times \mathbb{I}[w_{h}(s,a)\geq \delta ]\right\} .
$$
For any state  $s$  with  $\sum_{a\in \mathcal{A}}\widehat{d_h^{\mathrm{U}}} (s,a)w_h(s,a)\mathbb{I}[w_h(s,a)\geq \delta ] > 0$ , with the first-order optimality condition, we have
$$
\pi_ {h} ^ {\text {I S W - B C}} (a | s) = \frac {\widehat {d _ {h} ^ {\mathrm {U}}} (s , a) w _ {h} (s , a) \mathbb {I} [ w _ {h} (s , a) \geq \delta ]}{\sum_ {a \in \mathcal {A}} \widehat {d _ {h} ^ {\mathrm {U}}} (s , a) w _ {h} (s , a) \mathbb {I} [ w _ {h} (s , a) \geq \delta ]}.
$$
For an expert state  $s$  with  $d_h^{\pi^{\mathrm{E}}}(s) > 0$ , if  $(s, \pi_h^{\mathrm{E}}(s)) \in \mathcal{D}_h^{\mathrm{E}} \cup \mathcal{D}_h^{\mathrm{S},1}$ , we have that
$$
\langle \theta_ {h} ^ {\star}, \phi_ {h} (s, \pi_ {h} ^ {\mathrm {E}} (s)) \rangle > \langle \theta_ {h} ^ {\star}, \phi_ {h} (s, a) \rangle , \quad \forall (s, a) \in \mathcal {D} _ {h} ^ {\mathrm {S}, 2}.
$$
This is due to the first statement that  $\Delta_h(\theta_h^\star) > 0$  in this theorem. Recall that
$$
c _ {h} (s, a; \theta_ {h} ^ {\star}) = \frac {1}{1 + \exp (- \langle \phi_ {h} (s , a) , \theta_ {h} ^ {\star} \rangle)} \quad \text {a n d} \quad w _ {h} (s, a) = \frac {c _ {h} (s , a ; \theta_ {h} ^ {\star})}{1 - c _ {h} (s , a ; \theta_ {h} ^ {\star})}.
$$
We can further obtain that  $w_{h}(s,\pi_{h}^{\mathrm{E}}(s)) > w_{h}(s,a)$  for any  $(s,a)\in \mathcal{D}_h^{\mathrm{S},2}$ . This implies that we can find a  $\delta$  such that  $\mathbb{I}\left[w_h(s,\pi_h^{\mathrm{E}}(s))\geq \delta \right] = 1$  for any  $(s,\pi_h^{\mathrm{E}}(s))\in \mathcal{D}_h^{\mathrm{E}}\cup \mathcal{D}_h^{\mathrm{S},1}$  and  $\mathbb{I}\left[w_h(s,a)\geq \delta \right] = 0$  for any  $(s,a)\in \mathcal{D}_h^{\mathrm{S},2}$ . Based on the above analytical form of  $\pi^{\mathrm{ISW - BC}}$ ,
we have that  $\pi^{\mathrm{ISW - BC}}(\pi_h^{\mathrm{E}}(s)|s) = 1$  for any  $(s,\pi_h^{\mathrm{E}}(s))\in \mathcal{D}_h^{\mathrm{E}}\cup \mathcal{D}_h^{\mathrm{S},1}$ . In summary, for any state  $s$  with  $(s,\pi_h^{\mathrm{E}}(s))\in \mathcal{D}_h^{\mathrm{E}}\cup \mathcal{D}_h^{\mathrm{S},1}$ , we have that  $\pi_h^{\mathrm{ISW - BC}}(\pi_h^{\mathrm{E}}(s)|s) = 1$ .
With the above property of  $\pi^{\mathrm{ISW - BC}}$ , we proceed to analyze the policy value gap. According to [42, Lemma 4.3], we have
$$
V \left(\pi^ {\mathrm {E}}\right) - V \left(\pi^ {\mathrm {I S W - B C}}\right) \leq H \sum_ {h = 1} ^ {H} \mathbb {E} _ {s \sim d _ {h} ^ {\pi^ {\mathrm {E}}} (\cdot)} \left[ \operatorname {T V} \left(\pi_ {h} ^ {\mathrm {E}} (\cdot | s), \pi_ {h} ^ {\mathrm {I S W - B C}} (\cdot | s)\right) \right].
$$
Since  $\pi^{\mathrm{E}}$  is assumed to be deterministic, we have
$$
\begin{array}{l} V (\pi^ {\mathrm {E}}) - V (\pi^ {\mathrm {I S W - B C}}) \leq H \sum_ {h = 1} ^ {H} \mathbb {E} _ {s \sim d _ {h} ^ {\pi^ {\mathrm {E}}} (\cdot)} \left[ \mathbb {E} _ {a \sim \pi_ {h} ^ {\mathrm {I S W - B C}} (\cdot | s)} \left[ \mathbb {I} \left\{a \neq \pi_ {h} ^ {\mathrm {E}} (s) \right\} \right] \right] \\ \stackrel {(a)} {\leq} H \sum_ {h = 1} ^ {H} \mathbb {E} _ {s \sim d _ {h} ^ {\pi^ {\mathrm {E}}} (\cdot)} \left[ \mathbb {I} \left\{(s, \pi_ {h} ^ {\mathrm {E}} (s)) \notin \mathcal {D} _ {h} ^ {\mathrm {E}} \cup \mathcal {D} _ {h} ^ {\mathrm {S}, 1} \right\} \right] \\ \stackrel {(b)} {=} H \sum_ {h = 1} ^ {H} \mathbb {E} _ {s \sim d _ {h} ^ {\pi^ {\mathrm {E}}} (\cdot)} \left[ \mathbb {I} \left\{(s, \pi_ {h} ^ {\mathrm {E}} (s)) \notin \mathcal {D} _ {h} ^ {\mathrm {U}} \right\} \right]. \\ \end{array}
$$
Inequality  $(a)$  follows the property of  $\pi^{\mathrm{ISW - BC}}$  derived above. In particular, for any state  $s$  with  $(s,\pi_h^{\mathrm{E}}(s))\in \mathcal{D}_h^{\mathrm{E}}\cup \mathcal{D}_h^{\mathrm{S},1}$ , we have that  $\pi_h^{\mathrm{ISW - BC}}(\pi_h^{\mathrm{E}}(s)|s) = 1$ . Equation (b) holds due to the Assumption 2. In particular, for an expert state  $s$  that  $d_h^{\pi_h^{\mathrm{E}}}(\bar{s}) > 0$ , the events of  $(s,\pi_h^{\mathrm{E}}(s))\notin \mathcal{D}_h^{\mathrm{E}}\cup \mathcal{D}_h^{\mathrm{S},1}$  and  $(s,\pi_h^{\mathrm{E}}(s))\notin \mathcal{D}_h^{\mathrm{U}}$  are equivalent.
Moreover, we take the expectation over  $\mathcal{D}^{\mathrm{U}}$  on both sides and obtain that
$$
\begin{array}{l} \mathbb {E} \left[ V (\pi^ {\mathrm {E}}) - V (\pi^ {\mathrm {I S W - B C}}) \right] \leq H \sum_ {h = 1} ^ {H} \mathbb {E} _ {s \sim d _ {h} ^ {\pi^ {\mathrm {E}}} (\cdot)} \left[ \mathbb {P} \left((s, \pi_ {h} ^ {\mathrm {E}} (s)) \notin \mathcal {D} _ {h} ^ {\mathrm {U}}\right) \right] \\ = H \sum_ {h = 1} ^ {H} \sum_ {s \in S} d _ {h} ^ {\pi^ {\mathrm {E}}} (s) \mathbb {P} \left(\left(s, \pi_ {h} ^ {\mathrm {E}} (s)\right) \notin \mathcal {D} _ {h} ^ {\mathrm {U}}\right). \\ \end{array}
$$