SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathsf {L} _ {h} (r, x, y) \\ = \mathbb {1} _ {\mathrm {h} (x) = y} \ell_ {2} (\bar {r}, x, 0) + \sum_ {j = 1} ^ {n _ {e}} \bar {c} _ {j} (x, y) \ell_ {2} (\bar {r}, x, j) \\ = \mathbb {1} _ {\mathsf {h} (x) = y} \left(1 - \frac {e ^ {\bar {r} (x , 0)}}{\sum_ {y ^ {\prime} \in \mathcal {Y} \cup \{0 \}} e ^ {\bar {r} (x , y ^ {\prime})}}\right) + \sum_ {j = 1} ^ {n _ {e}} \bar {c} _ {j} (x, y) \left(1 - \frac {e ^ {\bar {r} (x , j)}}{\sum_ {y ^ {\prime} \in \mathcal {Y} \cup \{0 \}} e ^ {\bar {r} (x , y ^ {\prime})}}\right) \\ = \mathbb {1} _ {\mathfrak {h} (x) = y} \left[ 1 - \frac {1}{1 + \sum_ {i = 1} ^ {n _ {e}} e ^ {- r _ {i} (x)}} \right] + \sum_ {j = 1} ^ {n _ {e}} \bar {c} _ {j} (x, y) \left[ 1 - \frac {e ^ {- r _ {j} (x)}}{1 + \sum_ {i = 1} ^ {n _ {e}} e ^ {- r _ {i} (x)}} \right]. \\ \end{array}
$$
D Proof of  $\mathcal{H}$ -consistency bounds for score-based two-stage surrogate losses (Theorem 1)
$$
\begin{array}{l} \mathcal {E} _ {\ell_ {0 - 1}} \left(h _ {p}\right) - \mathcal {E} _ {\ell_ {0 - 1}} ^ {*} \left(\mathcal {H} _ {p}\right) + \mathcal {M} _ {\ell_ {0 - 1}} \left(\mathcal {H} _ {p}\right) \leq \Gamma_ {1} \left(\mathcal {E} _ {\ell_ {1}} \left(h _ {p}\right) - \mathcal {E} _ {\ell_ {1}} ^ {*} \left(\mathcal {H} _ {p}\right) + \mathcal {M} _ {\ell_ {1}} \left(\mathcal {H} _ {p}\right)\right) \\ \mathcal {E} _ {\ell_ {0 - 1}} (\overline {{h}} _ {d}) - \mathcal {E} _ {\ell_ {0 - 1}} ^ {*} (\overline {{\mathcal {H}}} _ {d}) + \mathcal {M} _ {\ell_ {0 - 1}} (\overline {{\mathcal {H}}} _ {d}) \leq \Gamma_ {2} \big (\mathcal {E} _ {\ell_ {2}} (\overline {{h}} _ {d}) - \mathcal {E} _ {\ell_ {2}} ^ {*} (\overline {{\mathcal {H}}} _ {d}) + \mathcal {M} _ {\ell_ {2}} (\overline {{\mathcal {H}}} _ {d}) \big). \\ \end{array}
$$
$$
\begin{array}{l} \mathcal {E} _ {\mathrm {L} _ {\text {d e f}}} (h) - \mathcal {E} _ {\mathrm {L} _ {\text {d e f}}} ^ {*} (\mathcal {H}) + \mathcal {M} _ {\mathrm {L} _ {\text {d e f}}} (\mathcal {H}) \\ \leq \Gamma_ {1} \Big (\mathcal {E} _ {\ell_ {1}} (h _ {p}) - \mathcal {E} _ {\ell_ {1}} ^ {*} (\mathcal {H} _ {p}) + \mathcal {M} _ {\ell_ {1}} (\mathcal {H} _ {p}) \Big) + \left(1 + \sum_ {j = 1} ^ {n _ {e}} \overline {{c}} _ {j}\right) \Gamma_ {2} \Bigg (\frac {\mathcal {E} _ {\mathsf {L} _ {h _ {p}}} (h _ {d}) - \mathcal {E} _ {\mathsf {L} _ {h _ {p}}} ^ {*} (\mathcal {H} _ {d}) + \mathcal {M} _ {\mathsf {L} _ {h _ {p}}} (\mathcal {H} _ {d})}{\sum_ {j = 1} ^ {n _ {e}} \underline {{c}} _ {j}} \Bigg). \\ \end{array}
$$
Furthermore, constant factors  $\left(1 + \sum_{j=1}^{n_e} \overline{c}_j\right)$  and  $\frac{1}{\sum_{j=1}^{n_e} c_j}$  can be removed when  $\Gamma_2$  is linear.
Proof. If  $\mathsf{h}(x) \in [n]$ , then  $\mathsf{h}(x) = \mathsf{h}_{\mathsf{p}}(x)$ . Thus, the learning to defer loss can be expressed as follows:
$$
\begin{array}{l} \mathsf {L} _ {\mathrm {d e f}} (h, x, y) = \mathbb {1} _ {\mathsf {h} (x) \neq y} \mathbb {1} _ {\mathsf {h} (x) \in [ n ]} + \sum_ {j = 1} ^ {n _ {e}} c _ {j} (x, y) \mathbb {1} _ {\mathsf {h} (x) = n + j} \\ = \mathbb {1} _ {\mathrm {h} _ {\mathrm {p}} (x) \neq y} \mathbb {1} _ {\mathrm {h} (x) \in [ n ]} + \sum_ {j = 1} ^ {n _ {e}} c _ {j} (x, y) \mathbb {1} _ {\mathrm {h} (x) = n + j}. \\ \end{array}
$$
Let  $\bar{c}_0(x,y) = \mathbb{1}_{\mathrm{hp}(x) = y}$ . Since  $h = (h_p,h_d)$ , we can rewrite  $\mathcal{E}_{\mathrm{L_{def}}}(h) - \mathcal{E}_{\mathrm{L_{def}}}^{*}(\mathcal{H}) + \mathcal{M}_{\mathrm{L_{def}}}(\mathcal{H})$  as
$$
\begin{array}{l} \mathcal {E} _ {\mathrm {L} _ {\text {d e f}}} (h) - \mathcal {E} _ {\mathrm {L} _ {\text {d e f}}} ^ {*} (\mathcal {H}) + \mathcal {M} _ {\mathrm {L} _ {\text {d e f}}} (\mathcal {H}) \\ = \mathbb {E} _ {X} \left[ \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} (h, x) - \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} ^ {*} (\mathcal {H}, x) \right] \\ = \mathbb {E} _ {X} \left[ \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} (h, x) - \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} (h, x) + \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} (h, x) - \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} ^ {*} (\mathcal {H}, x) \right] \tag {6} \\ = \mathbb {E} _ {X} \left[ \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} (h, x) - \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} (h, x) \right] + \mathbb {E} _ {X} \left[ \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} (h, x) - \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} ^ {*} (\mathcal {H}, x) \right]. \\ \end{array}
$$
Let  $\overline{p}(x, j) = \frac{\mathbb{E}_y[\bar{c}_j(x, y)]}{\mathbb{E}_y[\sum_{j=0}^{n_e} \bar{c}_j(x, y)]}$  for any  $j \in \{0, \dots, n_e\}$ . Note that  $\overline{p}(x, \cdot)$  is the probability vector on the label space  $\{0, \dots, n_e\}$ . For any  $h \in \mathcal{H}$ , we define  $\overline{h}$  as its augmented hypothesis:  $\overline{h}(x, 0) =$
$\max_{y \in \mathcal{Y}} h(x, y), \overline{h}(x, 1) = h(x, 1), \ldots, \overline{h}(x, n_e) = h(x, n_e)$ . By the assumptions, we have
$$
\mathcal {C} _ {\mathsf {L} _ {\text {d e f}}} (h, x) - \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathcal {C} _ {\mathsf {L} _ {\text {d e f}}} (h, x)
$$
$$
= \mathbb {E} _ {y} \left[ \mathbb {1} _ {\mathfrak {h} _ {\mathfrak {p}} (x) \neq y} \mathbb {1} _ {\mathfrak {h} (x) \in [ n ]} + \sum_ {j = 1} ^ {n _ {e}} c _ {j} (x, y) \mathbb {1} _ {\mathfrak {h} (x) = n + j} \right] - \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathbb {E} _ {y} \left[ \mathbb {1} _ {\mathfrak {h} _ {\mathfrak {p}} (x) \neq y} \mathbb {1} _ {\mathfrak {h} (x) \in [ n ]} + \sum_ {j = 1} ^ {n _ {e}} c _ {j} (x, y) \mathbb {1} \right. _ {\mathfrak {h} (x) = n + j}
$$
$$
= \mathbb {E} _ {y} \left[ \sum_ {j = 0} ^ {n _ {e}} \bar {c} _ {j} (x, y) \right] \times \left[ \sum_ {j = 0} ^ {n _ {e}} \bar {p} (x, j) \ell_ {0 - 1} (\bar {h}, x, j) - \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \sum_ {j = 0} ^ {n _ {e}} \bar {p} (x, j) \ell_ {0 - 1} (\bar {h}, x, j) \right]
$$
$$
\leq \mathbb {E} _ {y} \left[ \sum_ {j = 0} ^ {n _ {e}} \bar {c} _ {j} (x, y) \right] \times \Gamma_ {2} \left[ \sum_ {j = 0} ^ {n _ {e}} \bar {p} (x, j) \ell_ {2} (\bar {h}, x, j) - \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \sum_ {j = 0} ^ {n _ {e}} \bar {p} (x, j) \ell_ {2} (\bar {h}, x, j) \right]
$$
(By  $\overline{\mathcal{H}}_d$ -consistency bounds of  $\ell_{2}$  under assumption,  $\lambda = \max_{y\in \mathcal{Y}}h(x,y))$
$$
= \mathbb {E} _ {y} \left[ \sum_ {j = 0} ^ {n _ {e}} \bar {c} _ {j} (x, y) \right] \Gamma_ {2} \left(\frac {\mathbb {E} _ {y} \left[ \mathsf {L} _ {h _ {p}} \left(h _ {d} , x , y\right) \right] - \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathbb {E} _ {y} \left[ \mathsf {L} _ {h _ {p}} \left(h _ {d} , x , y\right) \right]}{\mathbb {E} _ {y} \left[ \sum_ {j = 0} ^ {n _ {e}} \bar {c} _ {j} (x , y) \right]}\right)
$$
$(\overline{p} (x,j) = \frac{\mathbb{E}_y[\bar{c}_j(x,y)]}{\mathbb{E}_y[\sum_{j = 0}^{n_e}\bar{c}_j(x,y)]},\overline{h} (x,0) = \max_{y\in \mathcal{Y}}h(x,y)$  and formulation (3))
$$
= \mathbb {E} _ {y} \left[ \sum_ {j = 0} ^ {n _ {e}} \bar {c} _ {j} (x, y) \right] \Gamma_ {2} \left(\frac {\mathcal {C} _ {\mathrm {L} _ {h _ {p}}} (h _ {d} , x) - \mathcal {C} _ {\mathrm {L} _ {h _ {p}}} ^ {*} (\mathcal {H} _ {d} , x)}{\mathbb {E} _ {y} \left[ \sum_ {j = 0} ^ {n _ {e}} \bar {c} _ {j} (x , y) \right]}\right)
$$
$$
\leq \left\{ \begin{array}{l l} \Gamma_ {2} \Big (\mathcal {C} _ {\mathsf {L} _ {h _ {p}}} (h _ {d}, x) - \mathcal {C} _ {\mathsf {L} _ {h _ {p}}} ^ {*} (\mathcal {H} _ {d}, x) \Big) & \text {w h e n} \Gamma_ {2} \text {i s l i n e a r} \\ \big (1 + \sum_ {j = 1} ^ {n _ {e}} \overline {{c}} _ {j} \big) \Gamma_ {2} \Bigg (\frac {\mathcal {C} _ {\mathsf {L} _ {h _ {p}}} (h _ {d} , x) - \mathcal {C} _ {\mathsf {L} _ {h _ {p}}} ^ {*} (\mathcal {H} _ {d} , x)}{\sum_ {j = 1} ^ {n _ {e}} \underline {{c}} _ {j}} \Bigg) & \text {o t h e r w i s e} \end{array} \right.
$$
$(\sum_{j=1}^{n_e} \underline{c}_j) \leq \mathbb{E}_y \left[ \sum_{j=0}^{n_e} \bar{c}_j(x, y) \right] \leq 1 + \sum_{j=1}^{n_e} \bar{c}_j$  and  $\Gamma_2$  is non-decreasing
$$
= \left\{ \begin{array}{l l} \Gamma_ {2} \Big (\Delta \mathcal {C} _ {\mathsf {L} _ {h _ {p}}, \mathcal {H} _ {d}} (h _ {d}, x) \Big) & \text {w h e n} \Gamma_ {2} \text {i s l i n e a r} \\ \big (1 + \sum_ {j = 1} ^ {n _ {e}} \overline {{c}} _ {j} \big) \Gamma_ {2} \Big (\frac {\Delta \mathcal {C} _ {\mathsf {L} _ {h _ {p}} , \mathcal {H} _ {d}} (h _ {d} , x)}{\sum_ {j = 1} ^ {n _ {e}} \underline {{c}} _ {- j}} \Big) & \text {o t h e r w i s e} \end{array} \right.
$$
$\leq \Gamma_{1}\bigl (\Delta \mathcal{C}_{\ell_{1},\mathcal{H}_{p}}(h_{p},x)\bigr).$  (By  $\mathcal{H}_p$  -consistency bounds of  $\ell$  under assumption)
$$
\begin{array}{l} \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathcal {C} _ {\mathsf {L} _ {\text {d e f}}} (h, x) - \mathcal {C} _ {\mathsf {L} _ {\text {d e f}}} ^ {*} (\mathcal {H}, x) \\ = \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} (h, x) - \inf  _ {h _ {p} \in \mathcal {H} _ {p}, h _ {d} \in \mathcal {H} _ {d}} \mathcal {C} _ {\mathrm {L} _ {\text {d e f}}} (h, x) \\ = \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathbb {E} _ {y} \left[ \mathbb {1} _ {\mathsf {h} _ {\mathfrak {p}} (x) \neq y} \mathbb {1} _ {\mathsf {h} (x) \in [ n ]} + \sum_ {j = 1} ^ {n _ {e}} c _ {j} (x, y) \mathbb {1} _ {\mathsf {h} (x) = n + j} \right] \\ - \inf  _ {h _ {p} \in \mathscr {H} _ {p}, h _ {d} \in \mathscr {H} _ {d}} \mathbb {E} _ {y} \left[ \mathbb {1} _ {\mathsf {h} _ {\mathsf {p}} (x) \neq y} \mathbb {1} _ {\mathsf {h} (x) \in [ n ]} + \sum_ {j = 1} ^ {n _ {e}} c _ {j} (x, y) \mathbb {1} _ {\mathsf {h} (x) = n + j} \right] \\ = \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathbb {E} _ {y} \left[ \mathbb {1} _ {\mathsf {h} _ {\mathfrak {p}} (x) \neq y} \mathbb {1} _ {\mathsf {h} (x) \in [ n ]} + \sum_ {j = 1} ^ {n _ {e}} c _ {j} (x, y) \mathbb {1} _ {\mathsf {h} (x) = n + j} \right] \\ - \inf  _ {h _ {d} \in \mathcal {H} _ {d}} \mathbb {E} _ {y} \left[ \inf  _ {h _ {p} \in \mathcal {H} _ {p}} \mathbb {1} _ {\mathsf {h} _ {\mathsf {p}} (x) \neq y} \mathbb {1} _ {\mathsf {h} (x) \in [ n ]} + \sum_ {j = 1} ^ {n _ {e}} c _ {j} (x, y) \mathbb {1} _ {\mathsf {h} (x) = n + j} \right] \\ = \min  \left\{\mathbb {E} _ {y} \left[ \mathbb {1} _ {\mathfrak {h} _ {\mathfrak {p}} (x) \neq y} \right], \min  _ {j \in [ p ]} \mathbb {E} _ {y} \left[ c _ {j} (x, y) \right] \right\} - \min  \left\{\inf  _ {h _ {p} \in \mathcal {H} _ {p}} \mathbb {E} _ {y} \left[ \mathbb {1} _ {\mathfrak {h} _ {\mathfrak {p}} (x) \neq y} \right], \min  _ {j \in [ p ]} \mathbb {E} _ {y} \left[ c _ {j} (x, y) \right] \right\} \\ \leq \mathbb {E} _ {y} \left[ \mathbb {1} _ {\mathrm {h} _ {\mathrm {p}} (x) \neq y} \right] - \inf  _ {h _ {p} \in \mathcal {H} _ {p}} \mathbb {E} _ {y} \left[ \mathbb {1} _ {\mathrm {h} _ {\mathrm {p}} (x) \neq y} \right] \\ = \mathcal {C} _ {\ell_ {0 - 1}} (h _ {p}, x) - \mathcal {C} _ {\ell_ {0 - 1}} ^ {*} (\mathcal {H} _ {p}, x) \\ = \Delta \mathcal {C} _ {\ell_ {0 - 1}, \mathcal {H} _ {p}} \left(h _ {p}, x\right) \\ \end{array}
$$
Therefore, by (6), we obtain
$$
\begin{array}{l} \mathcal {E} _ {\mathrm {L} _ {\text {d e f}}} (h) - \mathcal {E} _ {\mathrm {L} _ {\text {d e f}}} ^ {*} (\mathcal {H}) + \mathcal {M} _ {\mathrm {L} _ {\text {d e f}}} (\mathcal {H}) \\ \leq \left\{ \begin{array}{l l} \mathbb {E} _ {X} \Big [ \Gamma_ {2} \Big (\Delta \mathcal {C} _ {\mathsf {L} _ {h _ {p}}, \mathcal {H} _ {d}} (h _ {d}, x) \Big) \Big ] + \mathbb {E} _ {X} \Big [ \Gamma_ {1} \Big (\Delta \mathcal {C} _ {\ell_ {1}, \mathcal {H} _ {p}} (h _ {p}, x) \Big) \Big ] & \text {w h e n} \Gamma_ {2} \text {i s l i n e a r} \\ \big (1 + \sum_ {j = 1} ^ {n _ {e}} \bar {c} _ {j} \big) \mathbb {E} _ {X} \Big [ \Gamma_ {2} \Big (\frac {\Delta \mathcal {C} _ {\mathsf {L} _ {h _ {p}} , \mathcal {H} _ {d}} (h _ {d} , x)}{\sum_ {j = 1} ^ {n _ {e}} \underline {{c}} _ {j}} \Big) \Big ] + \mathbb {E} _ {X} \Big [ \Gamma_ {1} \Big (\Delta \mathcal {C} _ {\ell_ {1}, \mathcal {H} _ {p}} (h _ {p}, x) \Big) \Big ] & \text {o t h e r w i s e} \end{array} \right. \\ \leq \left\{ \begin{array}{l l} \Gamma_ {2} \Big (\mathbb {E} _ {X} \Big [ \Delta \mathcal {C} _ {\mathsf {L} _ {h _ {p}}, \mathcal {H} _ {d}} (h _ {d}, x) \Big ] \Big) + \Gamma_ {1} \Big (\mathbb {E} _ {X} \Big [ \Delta \mathcal {C} _ {\ell_ {1}, \mathcal {H} _ {p}} (h _ {p}, x) \Big ] \Big) & \text {w h e n} \Gamma_ {2} \text {i s l i n e a r} \\ \big (1 + \sum_ {j = 1} ^ {n _ {e}} \overline {{c}} _ {j} \big) \Gamma_ {2} \Bigg (\frac {1}{\sum_ {j = 1} ^ {n _ {e}} \underline {{c}} _ {j}} \mathbb {E} _ {X} \Big [ \Delta \mathcal {C} _ {\mathsf {L} _ {h _ {p}}, \mathcal {H} _ {d}} (h _ {d}, x) \Big ] \Bigg) + \Gamma_ {1} \Big (\mathbb {E} _ {X} \Big [ \Delta \mathcal {C} _ {\ell_ {1}, \mathcal {H} _ {p}} (h _ {p}, x) \Big ] \Big) & \text {o t h e r w i s e} \\ & (\Gamma_ {1} \text {a n d} \Gamma_ {2} \text {a r e c o n c a v e}) \end{array} \right. \\ = \left\{ \begin{array}{l l} \Gamma_ {1} \Big (\mathcal {E} _ {\ell_ {1}} (h _ {p}) - \mathcal {E} _ {\ell_ {1}} ^ {*} (\mathcal {H} _ {p}) + \mathcal {M} _ {\ell_ {1}} (\mathcal {H} _ {p}) \Big) + \Gamma_ {2} \Big (\mathcal {E} _ {\mathrm {L} _ {h _ {p}}} (h _ {d}) - \mathcal {E} _ {\mathrm {L} _ {h _ {p}}} ^ {*} (\mathcal {H} _ {d}) + \mathcal {M} _ {\mathrm {L} _ {h _ {p}}} (\mathcal {H} _ {d}) \Big) & \text {w h e n} \Gamma_ {2} \text {i s l i n e a r} \\ \Gamma_ {1} \Big (\mathcal {E} _ {\ell_ {1}} (h _ {p}) - \mathcal {E} _ {\ell_ {1}} ^ {*} (\mathcal {H} _ {p}) + \mathcal {M} _ {\ell_ {1}} (\mathcal {H} _ {p}) \Big) + \big (1 + \sum_ {j = 1} ^ {n _ {e}} \overline {{c}} _ {j} \big) \Gamma_ {2} \Bigg (\frac {\mathcal {E} _ {\mathrm {L} _ {h _ {p}}} (h _ {d}) - \mathcal {E} _ {\mathrm {L} _ {h _ {p}}} ^ {*} (\mathcal {H} _ {d}) + \mathcal {M} _ {\mathrm {L} _ {h _ {p}}} (\mathcal {H} _ {d})}{\sum_ {j = 1} ^ {n _ {e}} \underline {{c}} _ {j}} \Bigg) & \text {o t h e r w i s e}, \end{array} \right. \\ \end{array}
$$
E Proof of  $\overline{\mathcal{H}}$ -consistency bounds for standard surrogate loss functions (Theorem 3)
Recall that for a hypothesis  $h \colon \mathcal{X} \times \mathcal{Y} \to \mathbb{R}$ , we define  $\overline{h}$  as its augmented hypothesis:  $\overline{h}(\cdot, 0) = \lambda, \overline{h}(\cdot, 1) = h(x, 1), \dots, \overline{h}(\cdot, n) = h(x, n)$  with some constant  $\lambda \in \mathbb{R}$ . We define  $\overline{\mathcal{H}}$  as the hypothesis set that consists of all such augmented hypotheses of  $\mathcal{H}$ :  $\overline{\mathcal{H}} = \{\overline{h} : h \in \mathcal{H}\}$ . The prediction associated by  $\overline{h} \in \overline{\mathcal{H}}$  to an input  $x \in \mathcal{X}$  is denoted by  $\overline{h}(x)$  and defined as the element in  $\mathcal{Y} \cup \{0\}$  with the highest score,  $\overline{h}(x) = \operatorname*{argmax}_{y \in \mathcal{Y} \cup \{0\}} h(x, y)$ , with an arbitrary but fixed deterministic strategy for breaking ties. For any  $x \in \mathcal{X}$  and label space  $\mathcal{Y} \cup \{0\}$ , we will denote, by  $\overline{\mathsf{H}}(x)$  the set of labels generated by hypotheses in  $\overline{\mathcal{H}}$ :  $\overline{\mathsf{H}}(x) = \{\overline{\mathsf{h}}(x) : h \in \overline{\mathcal{H}}\}$ . By [Awasthi et al., 2022a, Lemma 3] with label space  $\mathcal{Y} \cup \{0\}$  and a conditional probability vector  $p(x, \cdot)$  on  $\mathcal{Y} \cup \{0\}$ , the minimal conditional  $\ell_{0-1}$ -loss and the corresponding calibration gap can be characterized as follows.
Lemma 8. For any  $x \in \mathcal{X}$ , the minimal conditional  $\ell_{0-1}$ -risk and the calibration gap for  $\ell_{0-1}$  can be expressed as follows:
$$
\mathcal{C}^{*}_{\ell_{0 - 1}}(x) = 1 - \max_{y\in \overline{\mathsf{H}} (x)}p(x,y)
$$
$$
\Delta \mathcal {C} _ {\ell_ {0 - 1}} (h, x) = \max  _ {y \in \overline {{\mathrm {H}}} (x)} p (x, y) - p (x, \mathsf {h} (x)).
$$
E.1 Multinomial logistic loss
Theorem 9 ( $\overline{\mathcal{H}}$ -consistency bound for multinomial logistic loss). Assume that  $\mathcal{H}$  is symmetric and complete. Then, for any  $\lambda \in \mathbb{R}$ , hypothesis  $\overline{h} \in \overline{\mathcal{H}}$  and any distribution,
$$
\mathcal {E} _ {\ell_ {0 - 1}} \left(\overline {{h}}\right) - \mathcal {E} _ {\ell_ {0 - 1}} ^ {*} \left(\overline {{\mathcal {H}}}\right) \leq \sqrt {2} \Big (\mathcal {E} _ {\ell_ {\log}} \left(\overline {{h}}\right) - \mathcal {E} _ {\ell_ {\log}} ^ {*} \left(\overline {{\mathcal {H}}}\right) + \mathcal {M} _ {\ell_ {\log}} \left(\overline {{\mathcal {H}}}\right) \Big) ^ {\frac {1}{2}} - \mathcal {M} _ {\ell_ {0 - 1}} \left(\overline {{\mathcal {H}}}\right).
$$
Proof. For the multinomial logistic loss  $\ell_{\log}$ , the conditional  $\ell_{\log}$ -risk can be expressed as follows:
$$
\mathcal {C} _ {\ell_ {\log}} (\bar {h}, x)) = \sum_ {y \in \mathcal {Y} \cup \{0 \}} p (x, y) \log \left(\sum_ {y ^ {\prime} \in \mathcal {Y} \cup \{0 \}} e ^ {\bar {h} (x, y ^ {\prime}) - \bar {h} (x, y)}\right) = - \sum_ {y \in \mathcal {Y} \cup \{0 \}} p (x, y) \log (\mathcal {S} (x, y))
$$
where we let  $\mathcal{S}(x,y) = \frac{e^{\overline{h}(x,y)}}{\sum_{y' \in \mathcal{Y} \cup \{0\}} e^{\overline{h}(x,y')}} \in [0,1]$  for any  $y \in \mathcal{Y} \cup \{0\}$  with the constraint that  $\sum_{y \in \mathcal{Y} \cup \{0\}} \mathcal{S}(x,y) = 1$ . Let  $y_{\max} = \operatorname{argmax}_{y \in \mathcal{Y} \cup \{0\}} p(x,y)$ , where we choose the label with the same deterministic strategy for breaking ties as that of  $\overline{h}(x)$ . For any  $\overline{h} \in \mathcal{H}$  such that  $\overline{h}(x) \neq y_{\max}$  and  $x \in \mathcal{X}$ , by the symmetry and completeness of  $\mathcal{H}$ , we can always find a family of hypotheses
$\{\overline{h}_{\mu}:\mu \in [-\mathcal{S}(x,y_{\max}),\mathcal{S}(x,\overline{\mathsf{h}} (x))]\} \subset \overline{\mathcal{H}}$  such that  $\mathcal{S}_{\mu}(x,\cdot) = \frac{e^{\overline{h}_{\mu}(x,\cdot)}}{\sum_{y'\in y\cup\{0\}}e^{\overline{h}_{\mu}(x,y')}}$  take the following values:
$$
\mathcal {S} _ {\mu} (x, y) = \left\{ \begin{array}{l l} \mathcal {S} (x, y) & \text {i f} y \notin \{y _ {\max }, \overline {{\mathsf {h}}} (x) \} \\ \mathcal {S} (x, y _ {\max }) + \mu & \text {i f} y = \overline {{\mathsf {h}}} (x) \\ \mathcal {S} (x, \overline {{\mathsf {h}}} (x)) - \mu & \text {i f} y = y _ {\max }. \end{array} \right.
$$
Note that  $\mathcal{S}_{\mu}$  satisfies the constraint:
$$
\sum_ {y \in \mathcal {Y} \cup \{0 \}} \mathcal {S} _ {\mu} (x, y) = \sum_ {y \in \mathcal {Y} \cup \{0 \}} \mathcal {S} (x, y) = 1, \forall \mu \in [ - \mathcal {S} (x, y _ {\max}), \mathcal {S} (x, \overline {{\mathsf {h}}} (x)) ].
$$
Let  $\overline{h} \in \overline{\mathcal{H}}$  be a hypothesis such that  $\overline{\mathsf{h}}(x) \neq y_{\max}$ . By the definition and using the fact that  $\overline{\mathsf{H}}(x) = \mathcal{Y} \cup \{0\}$  when  $\mathcal{H}$  is symmetric, we obtain
$$
\begin{array}{l} \Delta \mathcal {C} _ {\ell_ {\log}, \overline {{\mathcal {H}}}} (\overline {{h}}, x) \\ = \mathcal {C} _ {\ell_ {\log}} (\bar {h}, x) - \mathcal {C} _ {\ell_ {\log}} ^ {*} (\overline {{\mathcal {H}}}, x) \\ \geq \mathcal{C}_{\ell_{\log}}\big(\overline{h},x\big) - \inf_{\mu \in [-\mathcal{S}(x,y_{\max}),\mathcal{S}(x,\overline{h} (x))]}\mathcal{C}_{\ell_{\log}}\big(\overline{h}_{\mu},x\big) \\ = \sup  _ {\mu \in [ - \mathcal {S} (x, y _ {\max }), \mathcal {S} (x, \overline {{\mathrm {h}}} (x)) ]} \left\{p (x, y _ {\max }) \left[ - \log (\mathcal {S} (x, y _ {\max })) + \log \big (\mathcal {S} (x, \overline {{\mathrm {h}}} (x)) - \mu \big) \right] \right. \\ \left. \left. + p (x, \bar {h} (x)) \left[ - \log \left(\mathcal {S} (x, \bar {h} (x))\right)\right) + \log \left(\mathcal {S} (x, y _ {\max }) + \mu\right) \right] \right\} \\ \end{array}
$$
Differentiating with respect to  $\mu$  yields the optimum value  $\mu^{*} = \frac{p(x,\overline{h}(x))\mathcal{S}(x,\overline{h}(x)) - p(x,y_{\max})\mathcal{S}(x,y_{\max})}{p(x,y_{\max}) + p(x,\overline{h}(x))}$ . Plugging that value in the inequality gives:
$$
\begin{array}{l} \Delta \mathcal {C} _ {\ell_ {\log}, \overline {{\mathcal {H}}}} (\overline {{h}}, x) \geq p (x, y _ {\max}) \log \frac {[ \mathsf {S} (x , \overline {{\mathsf {h}}} (x)) + \mathsf {S} (x , y _ {\max}) ] p (x , y _ {\max})}{\mathsf {S} (x , y _ {\max}) [ p (x , y _ {\max}) + p (x , \overline {{\mathsf {h}}} (x)) ]} \\ + p (x, \overline {{\mathsf {h}}} (x)) \log \frac {\left[ \mathcal {S} (x , \overline {{\mathsf {h}}} (x)) + \mathcal {S} (x , y _ {\max }) \right] p (x , \overline {{\mathsf {h}}} (x))}{\mathcal {S} (x , \overline {{\mathsf {h}}} (x)) \left[ p (x , y _ {\max }) + p (x , \overline {{\mathsf {h}}} (x)) \right]}. \\ \end{array}
$$
Differentiating with respect to  $S$  to show that the minimum is attained for  $S(x, \overline{h}(x)) = S(x, y_{\max})$ , which implies
$$
\Delta \mathcal {C} _ {\ell_ {\log}, \overline {{\mathcal {H}}}} (\overline {{h}}, x) \geq p (x, y _ {\max}) \log \frac {2 p (x , y _ {\max})}{p (x , y _ {\max}) + p (x , \overline {{\mathsf {h}}} (x))} + p (x, \overline {{\mathsf {h}}} (x)) \log \frac {2 p (x , \overline {{\mathsf {h}}} (x))}{p (x , y _ {\max}) + p (x , \overline {{\mathsf {h}}} (x))}.
$$
By Pinsker's inequality, we have, for  $a, b \in [0,1]$ ,  $a\log \frac{2a}{a + b} + b\log \frac{2b}{a + b} \geq \frac{(a - b)^2}{2(a + b)}$ . Using this inequality, we obtain:
$$
\begin{array}{l} \Delta \mathcal {C} _ {\ell_ {\log}, \overline {{\mathcal {H}}}} (\bar {h}, x) \geq \frac {\left(p (x , \bar {h} (x)) - p (x , y _ {\max})\right) ^ {2}}{2 \left(p (x , \bar {h} (x)) + p (x , y _ {\max})\right)} \\ \geq \frac {\left(p (x , \overline {{\mathsf {h}}} (x)) - p (x , y _ {\max })\right) ^ {2}}{2} \quad (0 \leq p (x, \overline {{\mathsf {h}}} (x)) + p (x, y _ {\max }) \leq 1) \\ = \frac {1}{2} \left(\Delta \mathcal {C} _ {\ell_ {0 - 1}, \overline {{\mathcal {H}}}} (\bar {h}, x)\right) ^ {2}. \quad \text {(b y L e m m a 8 a n d} \overline {{\mathcal {H}}} (x) = \mathcal {Y} \cup \{0 \}) \\ \end{array}
$$
Since the function  $\frac{t^2}{2}$  is convex, by Jensen's inequality, we obtain for any hypothesis  $\overline{h} \in \overline{\mathcal{H}}$  and any distribution,
$$
\frac {\left(\mathbb {E} _ {X} \Big [ \Delta \mathcal {C} _ {\ell_ {0 - 1} , \overline {{\mathcal {H}}}} (\overline {{h}} , x) \Big ]\right) ^ {2}}{2} \leq \underset {X} {\mathbb {E}} \left[ \frac {\Delta \mathcal {C} _ {\ell_ {0 - 1} , \overline {{\mathcal {H}}}} (\overline {{h}} , x) ^ {2}}{2} \right] \leq \underset {X} {\mathbb {E}} \Big [ \Delta \mathcal {C} _ {\ell_ {\log}, \overline {{\mathcal {H}}}} (\overline {{h}}, x) \Big ],
$$
$$
\mathcal {E} _ {\ell_ {0 - 1}} (\overline {{h}}) - \mathcal {E} _ {\ell_ {0 - 1}} ^ {*} (\overline {{\mathcal {H}}}) \leq \sqrt {2} \Big (\mathcal {E} _ {\ell_ {\log}} (\overline {{h}}) - \mathcal {E} _ {\ell_ {\log}} ^ {*} (\overline {{\mathcal {H}}}) + \mathcal {M} _ {\ell_ {\log}} (\overline {{\mathcal {H}}}) \Big) ^ {\frac {1}{2}} - \mathcal {M} _ {\ell_ {0 - 1}} (\overline {{\mathcal {H}}}).
$$
E.2 Sum exponential loss
Theorem 10 ( $\overline{\mathcal{H}}$ -consistency bound for sum exponential loss). Assume that  $\mathcal{H}$  is symmetric and complete. Then, for any  $\lambda \in \mathbb{R}$ , hypothesis  $\overline{h} \in \overline{\mathcal{H}}$  and any distribution,
$$
\mathcal {E} _ {\ell_ {0 - 1}} (\overline {{h}}) - \mathcal {E} _ {\ell_ {0 - 1}} ^ {*} (\overline {{\mathcal {H}}}) \leq \sqrt {2} \big (\mathcal {E} _ {\ell_ {\exp}} (\overline {{h}}) - \mathcal {E} _ {\ell_ {\exp}} ^ {*} (\overline {{\mathcal {H}}}) + \mathcal {M} _ {\ell_ {\exp}} (\overline {{\mathcal {H}}}) \big) ^ {\frac {1}{2}} - \mathcal {M} _ {\ell_ {0 - 1}} (\overline {{\mathcal {H}}}).
$$
Proof. For the sum exponential loss  $\ell_{\mathrm{exp}}$ , the conditional  $\ell_{\mathrm{exp}}$ -risk can be expressed as follows:
$$
\mathcal {C} _ {\ell_ {\exp}} \big (\overline {{h}}, x \big) = \sum_ {y \in \mathcal {Y} \cup \{0 \}} p (x, y) \left(\sum_ {y ^ {\prime} \in \mathcal {Y} \cup \{0 \}} e ^ {\overline {{h}} (x, y ^ {\prime}) - \overline {{h}} (x, y)}\right) - 1 = \sum_ {y \in \mathcal {Y} \cup \{0 \}} \frac {p (x , y)}{\mathcal {S} (x , y)} - 1
$$