SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathbb {E} [ (\theta^ {\star} - \hat {\theta} _ {\mathrm {S L}}) ^ {2} ] = \mathbb {E} \left[ \left(\mathbb {E} [ Y ] - \frac {1}{m + n} \left(\sum_ {i = 1} ^ {m} \hat {f} (X _ {i}) + \sum_ {i = m + 1} ^ {m + n} Y _ {i}\right)\right) ^ {2} \right] \\ \leq 2 \left(\mathbb {E} \left[ \left(\frac {m}{m + n} \left(\mathbb {E} [ Y ] - \frac {1}{m} \sum_ {i = 1} ^ {m} \hat {f} (X _ {i})\right)\right) ^ {2} \right] + \mathbb {E} \left[ \left(\frac {n}{m + n} \left(\mathbb {E} [ Y ] - \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} Y _ {i}\right)\right) ^ {2} \right]\right) \\ \leq \frac {2 m ^ {2}}{(m + n) ^ {2}} \mathbb {E} [ (\hat {f} (X) - Y) ] ^ {2} + \frac {2 m}{(m + n) ^ {2}} \mathsf {V a r} [ \hat {f} (X) - Y ] + \frac {2 n}{(m + n) ^ {2}} \mathsf {V a r} [ Y ]. \\ \end{array}
$$
For the doubly robust loss, on one hand, we have
$$
\begin{array}{l} \mathbb {E} [ (\theta^ {\star} - \hat {\theta} _ {\mathsf {D R}}) ^ {2} ] = \mathbb {E} \left[ \left(\mathbb {E} [ Y ] - \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} \hat {f} (X _ {i}) + \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} (\hat {f} (X _ {i}) - Y _ {i})\right) ^ {2} \right] \\ \leq 2 \mathbb {E} \left[ \left(\mathbb {E} [ Y ] - \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} Y _ {i}\right) ^ {2} \right] + 2 \mathbb {E} \left[ \left(\mathbb {E} [ \hat {f} (X) ] - \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \hat {f} (X _ {i})\right) ^ {2} \right] \\ + 2 \mathbb {E} \left[ \left(\mathbb {E} [ \hat {f} (X) ] - \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} \hat {f} \left(X _ {i}\right)\right) ^ {2} \right] \\ = \frac {2}{n} \operatorname {V a r} [ Y ] + \left(\frac {2}{m + n} + \frac {2}{n}\right) \operatorname {V a r} [ \hat {f} (X) ]. \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} [ (\theta^ {\star} - \hat {\theta} _ {\mathrm {D R}}) ^ {2} ] = \mathbb {E} \left[ \left(\mathbb {E} [ Y ] - \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} \hat {f} (X _ {i}) + \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} (\hat {f} (X _ {i}) - Y _ {i})\right) ^ {2} \right] \\ \leq 2 \mathbb {E} \left[ \left(\mathbb {E} [ Y ] - \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} Y _ {i}\right) ^ {2} \right] + 2 \mathbb {E} \left[ \left(\mathbb {E} [ \hat {f} (X) - Y ] - \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} (\hat {f} (X _ {i}) - Y _ {i})\right) ^ {2} \right] \\ + 2 \mathbb {E} \left[ \left(\mathbb {E} [ \hat {f} (X) - Y ] - \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} (\hat {f} (X _ {i}) - Y _ {i})\right) ^ {2} \right] \\ = \left(\frac {2}{m + n} + \frac {2}{n}\right) \operatorname {V a r} [ \hat {f} (X) - Y ] + \frac {2}{m + n} \operatorname {V a r} [ Y ]. \\ \end{array}
$$
The proof is done by taking the minimum of the two upper bounds.
G Proof of Theorem 2
Proof. We know that
$$
\begin{array}{l} \left\| \nabla_ {\theta} \mathcal {L} _ {\mathcal {D} _ {1}, \mathcal {D} _ {2}} ^ {\mathsf {D R}} \left(\theta^ {\star}\right) - \mathbb {E} \left[ \nabla_ {\theta} \mathcal {L} _ {\mathcal {D} _ {1}, \mathcal {D} _ {2}} ^ {\mathsf {D R}} \left(\theta^ {\star}\right) \right] \right\| _ {2} \\ = \left\| \right. \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {*}} \left(X _ {i}, \hat {f} (X _ {i})\right) - \mathbb {E} \left[ \nabla_ {\theta} \ell_ {\theta^ {*}} \left(X, \hat {f} (X)\right)\right]\right) + \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {*}} \left(X _ {i}, Y _ {i}\right) - \nabla_ {\theta} \ell_ {\theta^ {*}} \left(X _ {i}, \hat {f} (X _ {i})\right)\right. \\ \left. - \mathbb {E} \left[ \nabla_ {\theta} \ell_ {\theta^ {\star}} (X, Y) - \nabla_ {\theta} \ell_ {\theta^ {\star}} (X, \hat {f} (X)) ]\right) \right\| _ {2} \\ \leq \left\| \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {*}} \left(X _ {i}, \hat {f} (X _ {i})\right) - \mathbb {E} \left[ \nabla_ {\theta} \ell_ {\theta^ {*}} \left(X, \hat {f} (X)\right) \right]\right) \right\| _ {2} + \left\| \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {*}} \left(X _ {i}, Y _ {i}\right) - \nabla_ {\theta} \ell_ {\theta^ {*}} \left(X _ {i}, \hat {f} (X _ {i})\right) \right. \right. \\ \left. \left. - \mathbb {E} \left[ \nabla_ {\theta} \ell_ {\theta^ {*}} (X, Y) - \nabla_ {\theta} \ell_ {\theta^ {*}} \bigl (X, \hat {f} (X) \bigr) \right]\right) \right\rVert_ {2}. \\ \end{array}
$$
From the multi-dimensional Chebyshev inequality (Bibby et al., 1979; Marshall and Olkin, 1960), we have that with probability at least  $1 - \delta /2$ , for some universal constant  $C$
$$
\left\| \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {*}} \left(X _ {i}, \hat {f} (X _ {i})\right) - \mathbb {E} \left[ \nabla_ {\theta} \ell_ {\theta^ {*}} \left(X, \hat {f} (X)\right) \right]\right) \right\| _ {2} \leq C \| \Sigma_ {\theta^ {*}} ^ {\hat {f}} \| _ {2} \sqrt {\frac {d}{(m + n) \delta}}.
$$
Similarly, we also have that with probability at least  $1 - \delta /2$
$$
\left\| \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \Big (\nabla_ {\theta} \ell_ {\theta^ {\star}} (X _ {i}, Y _ {i}) - \nabla_ {\theta} \ell_ {\theta^ {\star}} (X _ {i}, \hat {f} (X _ {i})) - \mathbb {E} [ \nabla_ {\theta} \ell_ {\theta^ {\star}} (X, Y) - \nabla_ {\theta} \ell_ {\theta^ {\star}} (X, \hat {f} (X)) ] \Big) \right\| _ {2} \leq C \| \Sigma_ {\theta^ {\star}} ^ {Y - \hat {f}} \| _ {2} \sqrt {\frac {d}{n \delta}}.
$$
$$
\mathbb {E} \left[ \nabla_ {\theta} \mathcal {L} _ {\mathcal {D} _ {1}, \mathcal {D} _ {2}} ^ {\mathsf {D R}} \left(\theta^ {\star}\right) \right] = \mathbb {E} \left[ \nabla_ {\theta} \ell_ {\theta^ {\star}} (X, Y) \right] = \nabla_ {\theta} \mathbb {E} \left[ \ell_ {\theta^ {\star}} (X, Y) \right] = 0.
$$
Here we use Assumption 1 and Assumption 2 to ensure that the expectation and differentiation are interchangeable. Thus we have that with probability at least  $1 - \delta$ ,
$$
\| \nabla_ {\theta} \mathcal {L} _ {\mathcal {D} _ {1}, \mathcal {D} _ {2}} ^ {\mathrm {D R}} (\theta^ {\star}) \| _ {2} \leq C \left(\| \Sigma_ {\theta^ {\star}} ^ {\hat {f}} \| _ {2} \sqrt {\frac {d}{(m + n) \delta}} + \| \Sigma_ {\theta^ {\star}} ^ {Y - \hat {f}} \| _ {2} \sqrt {\frac {d}{n \delta}}\right).
$$
On the other hand, we can also write the difference as
$$
\begin{array}{l} \| \nabla_ {\theta} \mathcal {L} _ {\mathcal {D} _ {1}, \mathcal {D} _ {2}} ^ {\mathsf {D R}} (\theta^ {\star}) - \mathbb {E} [ \nabla_ {\theta} \mathcal {L} _ {\mathcal {D} _ {1}, \mathcal {D} _ {2}} ^ {\mathsf {D R}} (\theta^ {\star}) ] \| _ {2} \\ = \Big \| \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {\star}} (X _ {i}, \hat {f} (X _ {i})) - \mathbb {E} [ \nabla_ {\theta} \ell_ {\theta^ {\star}} (X, \hat {f} (X)) ]\right) + \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {\star}} (X _ {i}, Y _ {i}) - \mathbb {E} [ \nabla_ {\theta} \ell_ {\theta^ {\star}} (X, Y) ]\right) \\ - \left. \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {*}} (X _ {i}, Y _ {i}) - \mathbb {E} \big [ \nabla_ {\theta} \ell_ {\theta^ {*}} (X, \hat {f} (X)) ]\right) \right\| _ {2} \\ \leq \left\| \frac {1}{m + n} \sum_ {i = 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {*}} (X _ {i}, \hat {f} (X _ {i})) - \mathbb {E} [ \nabla_ {\theta} \ell_ {\theta^ {*}} (X, \hat {f} (X)) ]\right) \right\| _ {2} + \left\| \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {*}} (X _ {i}, Y _ {i}) - \mathbb {E} [ \nabla_ {\theta} \ell_ {\theta^ {*}} (X, Y) ]\right) \right\| _ {2} \\ + \left\| \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \left(\nabla_ {\theta} \ell_ {\theta^ {*}} (X _ {i}, Y _ {i}) - \mathbb {E} [ \nabla_ {\theta} \ell_ {\theta^ {*}} (X, \hat {f} (X)) ]\right) \right\| _ {2} \\ \leq C \left(\| \Sigma_ {\theta^ {\star}} ^ {\hat {f}} \| _ {2} \left(\sqrt {\frac {d}{(m + n) \delta}} + \sqrt {\frac {d}{n \delta}}\right) + \| \Sigma_ {\theta^ {\star}} ^ {Y} \| _ {2} \sqrt {\frac {d}{n \delta}}\right). \\ \end{array}
$$
Here the last inequality uses the multi-dimensional Chebyshev inequality and it holds with probability at least  $1 - \delta$ . This finishes the proof.
H Proof of Proposition 3
$$
\begin{array}{l} \mathbb {E} [ \mathcal {L} _ {\mathcal {D} _ {1}, \mathcal {D} _ {2}} ^ {\mathsf {D R 2}} (\theta) ] = \frac {1}{m} \sum_ {i = 1} ^ {m} \mathbb {E} _ {X _ {i} \sim \mathbb {P} _ {X}} [ \ell_ {\theta} (X _ {i}, \hat {f} (X _ {i})) ] - \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \mathbb {E} _ {X _ {i} \sim \mathbb {Q} _ {X}} \left[ \frac {1}{\pi (X _ {i})} \ell_ {\theta} (X _ {i}, \hat {f} (X _ {i})) \right] \\ + \frac {1}{n} \sum_ {i = m + 1} ^ {m + n} \mathbb {E} _ {X _ {i} \sim \mathbb {Q} _ {X}, Y _ {i} \sim \mathbb {P} _ {Y | X _ {i}}} \left[ \frac {1}{\pi (X _ {i})} \ell_ {\theta} (X _ {i}, Y _ {i}) \right] \\ = \mathbb {E} _ {X \sim \mathbb {P} _ {X}} [ \ell_ {\theta} (X, \hat {f} (X)) ] - \mathbb {E} _ {X \sim \mathbb {Q} _ {X}} \left[ \frac {1}{\pi (X)} \ell_ {\theta} (X, \hat {f} (X)) \right] \\ + \mathbb {E} _ {X \sim \mathbb {Q} _ {X}, Y \sim \mathbb {P} _ {Y | X}} \left[ \frac {1}{\pi (X)} \ell_ {\theta} (X, Y) \right]. \\ \end{array}
$$
In the first case when  $\pi (x)\equiv \frac{\mathbb{P}_X(x)}{\mathbb{Q}_X(x)}$  , we have
$$
\begin{array}{l} \mathbb {E} \left[ \mathcal {L} _ {\mathcal {D} _ {1}, \mathcal {D} _ {2}} ^ {\mathrm {D R 2}} (\theta) \right] = \mathbb {E} _ {X \sim \mathbb {P} _ {X}} \left[ \ell_ {\theta} (X, \hat {f} (X)) \right] - \mathbb {E} _ {X \sim \mathbb {Q} _ {X}} \left[ \frac {\mathbb {P} _ {X} (X)}{\mathbb {Q} _ {X} (X)} \ell_ {\theta} (X, \hat {f} (X)) \right] \\ + \mathbb {E} _ {X \sim \mathbb {Q} _ {X}, Y \sim \mathbb {P} _ {Y | X}} \left[ \frac {\mathbb {P} _ {X} (X)}{\mathbb {Q} _ {X} (X)} \ell_ {\theta} (X, Y) \right] \\ = \mathbb {E} _ {X \sim \mathbb {P} _ {X}} \left[ \ell_ {\theta} (X, \hat {f} (X)) \right] - \mathbb {E} _ {X \sim \mathbb {P} _ {X}} \left[ \ell_ {\theta} (X, \hat {f} (X)) \right] \\ + \mathbb {E} _ {X \sim \mathbb {P} _ {X}, Y \sim \mathbb {P} _ {Y \mid X}} \left[ \ell_ {\theta} (X, Y) \right] \\ = \mathbb {E} _ {X, Y \sim \mathbb {P} _ {X, Y}} \left[ \ell_ {\theta} (X, Y) \right]. \\ \end{array}
$$
In the second case when  $\ell_{\theta}(x,\hat{f} (x)) = \mathbb{E}_{Y\sim \mathbb{P}_{Y|X = x}}[\ell_{\theta}(x,Y)]$  , we have
$$
\begin{array}{l} \mathbb {E} \left[ \mathcal {L} _ {\mathcal {D} _ {1}, \mathcal {D} _ {2}} ^ {\mathsf {D R 2}} (\theta) \right] = \mathbb {E} _ {X \sim \mathbb {P} _ {X}} \left[ \ell_ {\theta} (X, \hat {f} (X)) \right] - \mathbb {E} _ {X \sim \mathbb {Q} _ {X}} \left[ \frac {1}{\pi (X)} \ell_ {\theta} (X, \hat {f} (X)) \right] \\ + \mathbb {E} _ {X \sim \mathbb {Q} _ {X}} \mathbb {E} _ {Y \sim \mathbb {P} _ {Y | X}} \left[ \frac {1}{\pi (X)} \ell_ {\theta} (X, Y) \mid X \right] \\ = \mathbb {E} _ {X \sim \mathbb {P} _ {X}} \left[ \ell_ {\theta} (X, \hat {f} (X)) \right] - \mathbb {E} _ {X \sim \mathbb {Q} _ {X}} \left[ \frac {1}{\pi (X)} \ell_ {\theta} (X, \hat {f} (X)) \right] \\ + \mathbb {E} _ {X \sim \mathbb {Q} _ {X}} \left[ \frac {1}{\pi (X)} \ell_ {\theta} (X, \hat {f} (X)) \right] \\ = \mathbb {E} _ {X \sim \mathbb {P} _ {X}} \left[ \ell_ {\theta} (X, \hat {f} (X)) \right] \\ = \mathbb {E} _ {X, Y \sim \mathbb {P} _ {X, Y}} \left[ \ell_ {\theta} (X, Y) \right]. \\ \end{array}
$$