SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathbb {E} \left[ \mathbb {E} _ {H _ {i}} \left[ \left\| G ^ {t + 1} - \hat {G} ^ {t + 1} \right\| _ {2} ^ {2} \right] \right] = \frac {p _ {\text {o u t}}}{B _ {1}} \mathbb {E} \left[ \mathbb {E} _ {H _ {i}} \left[ \left\| G _ {i} ^ {t + 1} - \hat {G} _ {i} ^ {t + 1} \right\| _ {2} ^ {2} \right] \right] \\ \left. + \left(1 - p _ {\text {o u t}}\right) \left(\left\| G ^ {t} - \hat {G} ^ {t} \right\| _ {2} ^ {2} + \frac {1}{B _ {2}} \mathbb {E} \left[ \mathbb {E} _ {H _ {i}} \left[ \left\| G _ {i} ^ {t + 1} - G _ {i} ^ {t} - \hat {G} _ {i} ^ {t + 1} - \hat {G} _ {i} ^ {t} \right\| _ {2} ^ {2} \right] \right]\right)\right) \\ \leq \frac {p _ {\text {o u t}}}{B _ {1}} \frac {C _ {f} ^ {2} \sigma_ {g} ^ {2}}{m} + (1 - p _ {\text {o u t}}) \left\| G ^ {t} - \hat {G} ^ {t} \right\| _ {2} ^ {2} \\ + \frac {\left(1 - p _ {\text {o u t}}\right)}{B _ {2} m} \left(\mathbb {E} \left[ \left\| G _ {i} ^ {t + 1} - G _ {i} ^ {t} \right\| _ {2} ^ {2} \right]\right) \\ \leq \frac {p _ {\mathrm {o u t}}}{B _ {1}} \frac {C _ {f} ^ {2} \sigma_ {g} ^ {2}}{m} + (1 - p _ {\mathrm {o u t}}) \left\| G ^ {t} - \hat {G} ^ {t} \right\| _ {2} ^ {2} + \frac {(1 - p _ {\mathrm {o u t}})}{B _ {2} m} \left(\mathbb {E} [ \| G _ {i} ^ {t + 1} - G _ {i} ^ {t} \| _ {2} ^ {2} ]\right). \\ \end{array}
$$
Unroll the recursion gives
$$
\frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} \left[ \left\| G ^ {t + 1} - \hat {G} ^ {t + 1} \right\| _ {2} ^ {2} \right] \leq \frac {1}{B _ {1}} \frac {C _ {f} ^ {2} \sigma_ {g} ^ {2}}{m} + \frac {4 (1 - p _ {\text {o u t}})}{B _ {2} m p _ {\text {o u t}}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \| G _ {i} ^ {t + 1} - G _ {i} ^ {t} \| _ {2} ^ {2} ].
$$
Lemma 10 (Staleness) Define the staleness of iterates at time  $t$  as  $\Xi^t \coloneqq \frac{1}{n} \sum_{j=1}^{n} \| \pmb{x}^t - \phi_j^t \|_2^2$  and let  $G^{t+1}$  be the gradient estimate, then
$$
\frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \Xi^ {t} ] \leq \frac {6 n ^ {2}}{B _ {2} ^ {2}} \gamma^ {2} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \| G ^ {t + 1} \| _ {2} ^ {2} ]. \tag {23}
$$
Proof: Like previously (Lemma 5), let  $\mathbb{E}[\cdot ]$  denote the expectation conditioned on all previous randomness until  $t - 1$ . It is clear that  $\Xi^0 = 0$ , so we only consider  $t > 0$ . We upper bound  $\mathbb{E}[\Xi^t ]$  as follows,
$$
\mathbb{E}[\Xi^{t}] = (1 - p_{\mathrm{out}})\underbrace{\frac{1}{n}\sum_{j = 1}^{n}\mathbb{E}[\left\| \boldsymbol{x}^{t} - \boldsymbol{\phi}_{j}^{t}\right\|_{2}^{2}]}_{\text{if time $t$ takes $\mathcal{B}_{2}$}} + p_{\mathrm{out}}\underbrace{\frac{1}{n}\sum_{j = 1}^{n}\mathbb{E}[\left\| \boldsymbol{x}^{t} - \boldsymbol{\phi}_{j}^{t}\right\|_{2}^{2}]}_{\text{if time $t$ takes $\mathcal{B}_{1}(\boldsymbol{\phi}_{j}^{t} = \boldsymbol{x}^{t - 1})$}}.
$$
Then we can expand  $\mathbb{E}[\Xi^t]$  as follows
$$
\begin{array}{l} \mathbb {E} [ \Xi^ {t} ] = \frac {1 - p _ {\mathrm {o u t}}}{n} \sum_ {j = 1} ^ {n} \mathbb {E} [ \| \boldsymbol {x} ^ {t} - \boldsymbol {\phi} _ {j} ^ {t} \| _ {2} ^ {2} ] + \frac {p _ {\mathrm {o u t}}}{n} \sum_ {j = 1} ^ {n} \mathbb {E} [ \| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {t - 1} \| _ {2} ^ {2} ] \\ \leq \frac {1 - p _ {\text {o u t}}}{n} \sum_ {j = 1} ^ {n} \left((1 + \frac {1}{\beta}) \mathbb {E} _ {i} [ \| \boldsymbol {x} ^ {t - 1} - \phi_ {j} ^ {t} \| _ {2} ^ {2} ] + (1 + \beta) \| \boldsymbol {x} ^ {t - 1} - \boldsymbol {x} ^ {t} \| _ {2} ^ {2}\right) + p _ {\text {o u t}} \gamma^ {2} \mathbb {E} [ \| G ^ {t} \| _ {2} ^ {2} ] \\ \leq \frac {1}{n} \sum_ {j = 1} ^ {n} (1 + \frac {1}{\beta}) \mathbb {E} _ {i} [ \| \pmb {x} ^ {t - 1} - \phi_ {j} ^ {t} \| _ {2} ^ {2} ] + (1 + \beta) \gamma^ {2} \mathbb {E} [ \| G ^ {t} \| _ {2} ^ {2} ] \\ \end{array}
$$
where we use Cauchy-Schwarz inequality with coefficient  $\beta > 0$ . By the definition of  $\phi_j^t$ ,
$$
\begin{array}{l} \mathbb {E} \left[ \Xi^ {t} \right] \leq \frac {1}{n} \sum_ {j = 1} ^ {n} \left(1 + \frac {1}{\beta}\right) \left(\frac {n - B _ {2}}{n} \left\| \boldsymbol {x} ^ {t - 1} - \phi_ {j} ^ {t - 1} \right\| _ {2} ^ {2} + \frac {B _ {2}}{n} \left\| \boldsymbol {x} ^ {t - 1} - \boldsymbol {x} ^ {t - 1} \right\| _ {2} ^ {2}\right) + (1 + \beta) \gamma^ {2} \mathbb {E} \left[ \| G ^ {t} \| _ {2} ^ {2} \right] \\ = (1 + \frac {1}{\beta}) (1 - \frac {B _ {2}}{n}) \Xi^ {t - 1} + (1 + \beta) \gamma^ {2} \mathbb {E} [ \| G ^ {t} \| _ {2} ^ {2} ]. \\ \end{array}
$$
By taking  $\beta = 2n / B_2$ , we have that  $(1 + \frac{1}{\beta})(1 - \frac{B_2}{n}) \leq 1 - \frac{B_2}{2n}$  and thus
$$
\mathbb {E} [ \Xi^ {t} ] \leq (1 - \frac {B _ {2}}{2 n}) \Xi^ {t - 1} + (1 + \frac {2 n}{B _ {2}}) \gamma^ {2} \mathbb {E} [ \| G ^ {t} \| _ {2} ^ {2} ].
$$
Note that  $\mathbb{E}[\Xi^0] = 0$ .
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \Xi^ {t} ] \leq \frac {2 n}{B _ {2}} (1 + \frac {2 n}{B _ {2}}) \gamma^ {2} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \| G ^ {t + 1} \| _ {2} ^ {2} ] \\ \leq \frac {6 n ^ {2}}{B _ {2} ^ {2}} \gamma^ {2} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \| G ^ {t + 1} \| _ {2} ^ {2} ]. \\ \end{array}
$$
The following lemma describes how the inner variable changes inside the variance.
Lemma 11 Denote  $\mathcal{E}_y^{t + 1} \coloneqq \mathbb{E}\left[\left\| \boldsymbol{y}_i^{t + 1} - \mathbb{E}_{\eta | i}[g_\eta (\boldsymbol{x}^t)]\right\| _2^2\right]$  to be the error from inner variance and  $p_{out}T \leq 1$ . Then
$$
\frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {y} ^ {t + 1} \leq \frac {(1 - p _ {i n}) C _ {g} ^ {2}}{p _ {i n} S _ {2}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \Xi^ {t} + \frac {2 \sigma_ {g} ^ {2}}{S _ {1}}.
$$
Meanwhile,  $\mathcal{E}_y^1 = \mathbb{E}[\| \pmb {y}_i^1 -\mathbb{E}_{\eta |i}[g_\eta (\pmb{x}^0)]\| _2^2 ] = \frac{\sigma_g^2}{S_1}$
$$
\begin{array}{l} \mathcal {E} _ {y} ^ {t + 1} \leq p _ {\mathrm {i n}} \frac {\sigma_ {g} ^ {2}}{S _ {1}} + (1 - p _ {\mathrm {i n}}) \mathbb {E} _ {i} [ \mathbb {E} _ {\eta | i} [ \| \pmb {y} _ {i} ^ {t} - \mathbb {E} _ {\eta | i} [ g _ {\eta} (\pmb {\phi} _ {i} ^ {t}) ] \| _ {2} ^ {2} ] ] \\ + \frac {1 - p _ {\mathrm {i n}}}{S _ {2}} \mathbb {E} _ {i} [ \mathbb {E} _ {\eta | i} [ \| g _ {\eta} (\boldsymbol {x} ^ {t}) - g _ {\eta} (\phi_ {i} ^ {t}) \| _ {2} ^ {2} ] ] \\ \leq (1 - p _ {\mathrm {i n}}) \mathcal {E} _ {y} ^ {t} + \frac {(1 - p _ {\mathrm {i n}}) C _ {g} ^ {2}}{S _ {2}} \mathbb {E} _ {i} [ \mathbb {E} _ {\eta | i} [ \| \pmb {x} ^ {t} - \pmb {\phi} _ {i} ^ {t} \| _ {2} ^ {2} ] ] + p _ {\mathrm {i n}} \frac {\sigma_ {g} ^ {2}}{S _ {1}}. \\ \end{array}
$$
As  $t = 0$  always uses the large batch,  $\mathcal{E}_y^1 = \mathbb{E}[\| \pmb {y}_i^1 -\mathbb{E}_{\eta |i}[g_\eta (\pmb{x}^0)]\| _2^2 ] = \frac{\sigma_g^2}{S_1}$ . Then
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {y} ^ {t + 1} \leq \frac {\left(1 - p _ {\mathrm {i n}}\right) C _ {g} ^ {2}}{p _ {\mathrm {i n}} S _ {2}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} _ {i} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \boldsymbol {x} ^ {t} - \phi_ {i} ^ {t} \right\| _ {2} ^ {2} \right] \right] + \frac {\sigma_ {g} ^ {2}}{S _ {1}} + \frac {\mathcal {E} _ {y} ^ {1}}{p _ {\mathrm {i n}} T} \\ \leq \frac {\left(1 - p _ {\mathrm {i n}}\right) C _ {g} ^ {2}}{p _ {\mathrm {i n}} S _ {2}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \Xi^ {t} + \frac {2 \sigma_ {g} ^ {2}}{S _ {1}}. \\ \end{array}
$$
Lemma 12 The error  $\mathbb{E}_i[\mathbb{E}_{p_{in}}[\mathbb{E}_{\eta |i}[\| \boldsymbol{y}_i^{t + 1} - \tilde{\boldsymbol{y}}_i^t\| _2^2 ]]]$  satisfies
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 1} ^ {T - 1} \mathbb {E} _ {i} [ \mathbb {E} _ {p _ {i n}} [ \mathbb {E} _ {\eta | i} [ \| \boldsymbol {y} _ {i} ^ {t + 1} - \tilde {\boldsymbol {y}} _ {i} ^ {t} \| _ {2} ^ {2} ] ] ] \leq \frac {4 C _ {g} ^ {2} \gamma^ {2}}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \| G ^ {t + 1} \| _ {2} ^ {2} ] + \frac {4 (1 - p _ {i n}) C _ {g} ^ {2}}{S _ {2}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \Xi^ {t + 1} \\ + \frac {6 (1 - p _ {i n})}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {y} ^ {t + 1}. \\ \end{array}
$$
Note that when  $p_{\mathrm{in}} = 1$  and  $S_{1} = S_{2} = m$ , we recover the following
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 1} ^ {T - 1} \mathbb {E} _ {i} [ \mathbb {E} _ {p _ {\mathrm {i n}}} [ \mathbb {E} _ {\eta | i} [ \| \pmb {y} _ {i} ^ {t + 1} - \tilde {\pmb {y}} _ {i} ^ {t} \| _ {2} ^ {2} ] ] ] = \frac {1}{T} \sum_ {t = 1} ^ {T - 1} \mathbb {E} _ {i} [ \mathbb {E} _ {p _ {\mathrm {i n}}} [ \mathbb {E} _ {\eta | i} [ \| \frac {1}{m} \sum_ {\eta \in H _ {\xi}} g _ {\eta} (\pmb {x} ^ {t}) - g _ {\eta} (\pmb {x} ^ {t - 1}) \| _ {2} ^ {2} ] ] ] ] \\ \leq \frac {4 C _ {g} ^ {2} \gamma^ {2}}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \| G ^ {t + 1} \| _ {2} ^ {2} ]. \\ \end{array}
$$
Proof: For  $t \geq 2$ ,  $\mathbb{E}_i[\mathbb{E}_{p_{\mathrm{in}}}[\mathbb{E}_{\eta |i}[\| \boldsymbol{y}_i^{t + 1} - \tilde{\boldsymbol{y}}_i^t\| _2^2 ]]]$  can be upper bounded as follows
$$
\mathbb {E} _ {i} \left[ \mathbb {E} _ {p _ {\mathrm {i n}}} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \boldsymbol {y} _ {i} ^ {t + 1} - \tilde {\boldsymbol {y}} _ {i} ^ {t} \right\| _ {2} ^ {2} \right] \right] \right] = p _ {\mathrm {i n}} \mathbb {E} _ {i} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \frac {1}{S _ {1}} \sum_ {\eta \in H _ {i}} \left(g _ {\eta} (\boldsymbol {x} ^ {t}) - g _ {\eta} (\boldsymbol {x} ^ {t - 1})\right) \right\| _ {2} ^ {2} \right] \right]
$$
$$
+ \left(1 - p _ {\mathrm {i n}}\right) \mathbb {E} _ {i} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \boldsymbol {y} _ {i} ^ {t} - \boldsymbol {y} _ {i} ^ {t - 1} + \frac {1}{S _ {2}} \sum_ {\eta \in H _ {i}} \left(g _ {\eta} (\boldsymbol {x} ^ {t}) - g _ {\eta} (\phi_ {i} ^ {t})\right) - \left(g _ {\eta} (\boldsymbol {x} ^ {t - 1}) - g _ {\eta} (\phi_ {i} ^ {t - 1})\right) \right\| _ {2} ^ {2} \right] \right]
$$
$$
\begin{array}{l} \leq p _ {\text {i n}} C _ {g} ^ {2} \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {t - 1} \right\| _ {2} ^ {2} + \frac {1 - p _ {\text {i n}}}{S _ {2}} \mathbb {E} _ {i} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \left(g _ {\eta} \left(\boldsymbol {x} ^ {t}\right) - g _ {\eta} \left(\phi_ {i} ^ {t}\right)\right) - \left(g _ {\eta} \left(\boldsymbol {x} ^ {t - 1}\right) - g _ {\eta} \left(\phi_ {i} ^ {t - 1}\right)\right) \right\| _ {2} ^ {2} \right] \right] \\ + 3 \left(1 - p _ {\mathrm {i n}}\right) \left(\mathbb {E} _ {i} \left[ \left\| \boldsymbol {y} _ {i} ^ {t} - \mathbb {E} _ {\eta | i} \left[ g _ {\eta} \left(\phi_ {i} ^ {t}\right) \right] \right\| _ {2} ^ {2} \right] + \mathbb {E} _ {i} \left[ \left\| \boldsymbol {y} _ {i} ^ {t - 1} - \mathbb {E} _ {\eta | i} \left[ g _ {\eta} \left(\phi_ {i} ^ {t - 1}\right) \right] \right\| _ {2} ^ {2} \right] + C _ {g} ^ {2} \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {t - 1} \right\| _ {2} ^ {2}\right) \\ \end{array}
$$
$$
\begin{array}{l} \leq p _ {\text {i n}} C _ {g} ^ {2} \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {t - 1} \right\| _ {2} ^ {2} + \frac {2 \left(1 - p _ {\text {i n}}\right) C _ {g} ^ {2}}{S _ {2}} \left(\Xi^ {t} + \Xi^ {t - 1}\right) + 3 \left(1 - p _ {\text {i n}}\right) \left(\mathcal {E} _ {y} ^ {t} + \mathcal {E} _ {y} ^ {t - 1} + C _ {g} ^ {2} \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {t - 1} \right\| _ {2} ^ {2}\right) \\ \leq \left(p _ {\text {i n}} + 3 \left(1 - p _ {\text {i n}}\right)\right) C _ {g} ^ {2} \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {t - 1} \right\| _ {2} ^ {2} + \frac {2 \left(1 - p _ {\text {i n}}\right) C _ {g} ^ {2}}{S _ {2}} \left(\Xi^ {t} + \Xi^ {t - 1}\right) + 3 \left(1 - p _ {\text {i n}}\right) \left(\mathcal {E} _ {y} ^ {t} + \mathcal {E} _ {y} ^ {t - 1}\right). \\ \end{array}
$$
For  $t = 1$ , we choose  $\tilde{\pmb{y}}_i^1 = \pmb{y}_i^1$
$$
\begin{array}{l} \mathbb {E} _ {i} \left[ \mathbb {E} _ {p _ {\mathrm {i n}}} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \boldsymbol {y} _ {i} ^ {2} - \tilde {\boldsymbol {y}} _ {i} ^ {1} \right\| _ {2} ^ {2} \right] \right] \right] = p _ {\mathrm {i n}} \mathbb {E} _ {i} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \frac {1}{S _ {1}} \sum_ {\eta \in H _ {i}} \left(g _ {\eta} (\boldsymbol {x} ^ {1}) - g _ {\eta} (\boldsymbol {x} ^ {0})\right) \right\| _ {2} ^ {2} \right] \right] \\ \left. \right. + \left(1 - p _ {\mathrm {i n}}\right) \mathbb {E} _ {i} \left[ \mathbb {E} _ {\eta | i} \left[\left\| \boldsymbol {y} _ {i} ^ {1} - \frac {1}{S _ {2}} \sum_ {\eta \in H _ {i}} \left(g _ {\eta} \left(\boldsymbol {x} ^ {1}\right) - g _ {\eta} \left(\boldsymbol {x} ^ {0}\right)\right) - \tilde {\boldsymbol {y}} _ {i} ^ {1} \right\| _ {2} ^ {2} \right]\right] \\ \leq C _ {g} ^ {2} \left\| \boldsymbol {x} ^ {1} - \boldsymbol {x} ^ {0} \right\| _ {2} ^ {2}. \\ \end{array}
$$
Then for summing up  $t = 1$  to  $T - 1$
$$
\begin{array}{l} \sum_ {t = 2} ^ {T - 1} \mathbb {E} _ {i} \left[ \mathbb {E} _ {p _ {\text {i n}}} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \boldsymbol {y} _ {i} ^ {t + 1} - \tilde {\boldsymbol {y}} _ {i} ^ {t} \right\| _ {2} ^ {2} \right] \right] \right] + \mathbb {E} _ {i} \left[ \mathbb {E} _ {p _ {\text {i n}}} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \boldsymbol {y} _ {i} ^ {2} - \tilde {\boldsymbol {y}} _ {i} ^ {1} \right\| _ {2} ^ {2} \right] \right] \right] \\ \leq \left(p _ {\text {i n}} + 3 \left(1 - p _ {\text {i n}}\right)\right) C _ {g} ^ {2} \sum_ {t = 2} ^ {T - 1} \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {t - 1} \right\| _ {2} ^ {2} + \frac {2 \left(1 - p _ {\text {i n}}\right) C _ {g} ^ {2}}{S _ {2}} \left(\sum_ {t = 2} ^ {T - 1} \Xi^ {t} + \sum_ {t = 2} ^ {T - 1} \Xi^ {t - 1}\right) \\ + 3 \left(1 - p _ {\mathrm {i n}}\right) \left(\sum_ {t = 2} ^ {T - 1} \mathcal {E} _ {y} ^ {t} + \sum_ {t = 2} ^ {T - 1} \mathcal {E} _ {y} ^ {t - 1}\right) + C _ {g} ^ {2} \left\| \boldsymbol {x} ^ {1} - \boldsymbol {x} ^ {0} \right\| _ {2} ^ {2} \\ \leq 4 C _ {g} ^ {2} \sum_ {t = 1} ^ {T - 1} \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {t - 1} \right\| _ {2} ^ {2} + \frac {4 (1 - p _ {\mathrm {i n}}) C _ {g} ^ {2}}{S _ {2}} \sum_ {t = 0} ^ {T - 1} \Xi^ {t + 1} + 6 (1 - p _ {\mathrm {i n}}) \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {y} ^ {t + 1}. \\ \end{array}
$$
Finally, the error has the following upper bound
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 1} ^ {T - 1} \mathbb {E} _ {i} \left[ \mathbb {E} _ {p _ {\mathrm {i n}}} \left[ \mathbb {E} _ {\eta | i} \left[ \left\| \boldsymbol {y} _ {i} ^ {t + 1} - \tilde {\boldsymbol {y}} _ {i} ^ {t} \right\| _ {2} ^ {2} \right] \right] \right] \\ \leq \frac {4 C _ {g} ^ {2} \gamma^ {2}}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \| G ^ {t + 1} \| _ {2} ^ {2} ] + \frac {4 (1 - p _ {\mathrm {i n}}) C _ {g} ^ {2}}{S _ {2}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \Xi^ {t + 1} + \frac {6 (1 - p _ {\mathrm {i n}})}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {y} ^ {t + 1}. \\ \end{array}
$$
Lemma 13 (Bias and Variance of NestedVR) If the step size  $\gamma$  satisfies,
$$
\gamma^ {2} L _ {F} ^ {2} \max  \left\{\frac {(1 - p _ {i n})}{p _ {i n} S _ {2}}, \frac {1 8}{B _ {2}}, \frac {1 - p _ {o u t}}{B _ {2}} \frac {(1 - p _ {i n})}{p _ {i n} S _ {2}} \frac {1 8 n ^ {2}}{B _ {2} ^ {2}}, \frac {(1 - p _ {o u t})}{B _ {2}} \right\} \leq \frac {1}{1 6} \cdot \frac {1}{6}
$$
then the variance and bias of NestedVR are
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {v a r} ^ {t + 1} \leq 3 2 \left(\left(\frac {p _ {o u t}}{B _ {1}} + \frac {1 - p _ {o u t}}{B _ {2}}\right) \frac {(1 - p _ {i n})}{p _ {i n} S _ {2}} \frac {1 8 n ^ {2}}{B _ {2} ^ {2}} + \frac {(1 - p _ {o u t})}{B _ {2}}\right) \frac {\gamma^ {2} L _ {F} ^ {2}}{T} \sum_ {t = 0} ^ {T - 1} \left\| \mathbb {E} [ G ^ {t + 1} ] \right\| _ {2} ^ {2} \\ + 9 6 \left(\frac {p _ {\text {o u t}}}{B _ {1}} + \frac {(1 - p _ {\text {i n}}) (1 - p _ {\text {o u t}})}{B _ {2}}\right) \frac {\tilde {L} _ {F} ^ {2}}{S _ {1}} + \frac {(1 - p _ {\text {o u t}})}{T} \frac {8 \tilde {L} _ {F} ^ {2}}{B _ {1} S _ {1}} \\ \end{array}
$$
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {b i a s} ^ {t + 1} \leq \frac {1 2 (1 - p _ {i n})}{p _ {i n} S _ {2}} \frac {n ^ {2}}{B _ {2} ^ {2}} \frac {L _ {F} ^ {2} \gamma^ {2}}{T} \sum_ {t = 0} ^ {T - 1} \left\| \mathbb {E} [ G ^ {t + 1} ] \right\| _ {2} ^ {2} + \frac {4 \tilde {L} _ {F} ^ {2}}{S _ {1}} \\ + \left(\frac {1 2 (1 - p _ {i n})}{p _ {i n} S _ {2}} \frac {n ^ {2}}{B _ {2} ^ {2}} L _ {F} ^ {2} \gamma^ {2} + \frac {2 (1 - p _ {o u t}) ^ {2}}{p _ {o u t}}\right) \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {\text {v a r}} ^ {t + 1}. \\ \end{array}
$$
Proof: Notations. Let us define the following terms,
$$
G _ {i} ^ {t + 1} := \left(\boldsymbol {z} _ {i} ^ {t + 1}\right) ^ {\top} \nabla f _ {i} \left(\boldsymbol {y} _ {i} ^ {t + 1}\right), \quad \tilde {G} _ {i} ^ {t} := \left(\boldsymbol {z} _ {i} ^ {t}\right) ^ {\top} \nabla f _ {i} \left(\tilde {\boldsymbol {y}} _ {i} ^ {t}\right).
$$
Note that the  $\tilde{G}^t$  computed at time  $t + 1$  has same expectation as  $G^t$
$$
\mathbb {E} ^ {t + 1} \left[ \tilde {G} ^ {t} | t \right] = \mathbb {E} ^ {t} \left[ G ^ {t} | t - 1 \right]. \tag {24}
$$
Computing the bias. First consider the two cases in the outer loop
$$
\begin{array}{l} \mathcal {E} _ {\mathrm {b i a s}} ^ {t + 1} = \left\| \nabla F (\boldsymbol {x} ^ {t}) - \mathbb {E} ^ {t + 1} [ G ^ {t + 1} | t ] \right\| _ {2} ^ {2} \\ \leq 2 \underbrace {\left\| \nabla F (\boldsymbol {x} ^ {t}) - \mathbb {E} ^ {t + 1} [ G _ {i} ^ {t + 1} | t ] \right\| _ {2} ^ {2}} _ {A _ {1} ^ {t + 1}} + 2 \underbrace {\left\| \mathbb {E} ^ {t + 1} [ G _ {i} ^ {t + 1} | t ] - \mathbb {E} ^ {t + 1} [ G ^ {t + 1} | t ] \right\| _ {2} ^ {2}} _ {A _ {2} ^ {t + 1}}. \\ \end{array}
$$
We expand  $A_2^{t+1}$  as follows
$$
\begin{array}{l} A _ {2} ^ {t + 1} = \left\| \mathbb {E} ^ {t + 1} \left[ G _ {i} ^ {t + 1} | t \right] - \mathbb {E} ^ {t + 1} \left[ G ^ {t + 1} | t \right] \right\| _ {2} ^ {2} \\ = \left\| \mathbb {E} ^ {t + 1} \left[ G _ {i} ^ {t + 1} | t \right] - p _ {\text {o u t}} \mathbb {E} ^ {t + 1} \left[ G _ {i} ^ {t + 1} | t \right] - \left(1 - p _ {\text {o u t}}\right) \left(G ^ {t} + \mathbb {E} ^ {t + 1} \left[ G _ {i} ^ {t + 1} - \tilde {G} _ {i} ^ {t} | t \right]\right) \right\| _ {2} ^ {2} \\ = (1 - p _ {\mathrm {o u t}}) ^ {2} \left\| G ^ {t} - \mathbb {E} ^ {t + 1} [ \tilde {G} _ {i} ^ {t} | t ] \right\| _ {2} ^ {2} \\ = \left(1 - p _ {\text {o u t}}\right) ^ {2} \left\| G ^ {t} - \mathbb {E} ^ {t} \left[ G _ {i} ^ {t} \mid t - 1 \right] \right\| _ {2} ^ {2} \\ \end{array}
$$
where we use (24) in the last equality. Now we take expectation with respect to randomness at  $t$  such that  $G^{t}$  is a random variable, then
$$
\begin{array}{l} A _ {2} ^ {t + 1} = (1 - p _ {\mathrm {o u t}}) ^ {2} \mathbb {E} ^ {t} \left[ \left\| G ^ {t} - \mathbb {E} ^ {t} [ G _ {i} ^ {t} | t - 1 ] \right\| _ {2} ^ {2} | t - 1 \right] \\ = \left(1 - p _ {\text {o u t}}\right) ^ {2} \left(\left\| \mathbb {E} ^ {t} [ G ^ {t} | t - 1 ] - \mathbb {E} ^ {t} [ G _ {i} ^ {t} | t - 1 ] \right\| _ {2} ^ {2} + \mathcal {E} _ {\text {v a r}} ^ {t}\right) \\ = \left(1 - p _ {\text {o u t}}\right) ^ {2} \left(A _ {2} ^ {t} + \mathcal {E} _ {\text {v a r}} ^ {t}\right) \\ \end{array}
$$
while at initialization we always use large batch
$$
A _ {2} ^ {1} = \left\| \mathbb {E} ^ {1} \left[ G _ {i} ^ {1} \right] - \mathbb {E} ^ {1} \left[ G ^ {1} \right] \right\| _ {2} ^ {2} = \left\| \mathbb {E} ^ {1} \left[ G _ {i} ^ {1} \right] - \mathbb {E} ^ {1} \left[ G _ {i} ^ {1} \right] \right\| _ {2} ^ {2} = 0.
$$
Therefore, when we average over time  $t$
$$
\frac {1}{T} \sum_ {t = 0} ^ {T - 1} A _ {2} ^ {t + 1} \leq \frac {\left(1 - p _ {\text {o u t}}\right) ^ {2}}{p _ {\text {o u t}}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {\text {v a r}} ^ {t + 1}. \tag {25}
$$
On the other hand, let us consider the upper bound on  $A_1^{t + 1}$
$$
A _ {1} ^ {t + 1} \leq C _ {g} ^ {2} L _ {f} ^ {2} \mathbb {E} [ \| \boldsymbol {y} _ {i} ^ {t + 1} - \mathbb {E} _ {\eta | i} [ g _ {\eta} (\boldsymbol {x} ^ {t}) ] \| _ {2} ^ {2} ] = C _ {g} ^ {2} L _ {f} ^ {2} \mathcal {E} _ {y} ^ {t + 1}.
$$
From Lemma 11 we know that
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} A _ {1} ^ {t + 1} \leq C _ {g} ^ {2} L _ {f} ^ {2} \left(\frac {\left(1 - p _ {\mathrm {i n}}\right) C _ {g} ^ {2}}{p _ {\mathrm {i n}} S _ {2}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \Xi^ {t} + \frac {2 \sigma_ {g} ^ {2}}{S _ {1}}\right) \\ \leq \frac {\left(1 - p _ {\mathrm {i n}}\right) L _ {F} ^ {2}}{p _ {\mathrm {i n}} S _ {2}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \Xi^ {t} + \frac {2 \tilde {L} _ {F} ^ {2}}{S _ {1}}. \\ \end{array}
$$
From Lemma 10 we know that
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} A _ {1} ^ {t + 1} \leq \frac {(1 - p _ {\mathrm {i n}}) L _ {F} ^ {2}}{p _ {\mathrm {i n}} S _ {2}} \left(\frac {6 n ^ {2}}{B _ {2} ^ {2}} \gamma^ {2} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \| G ^ {t + 1} \| _ {2} ^ {2} ]\right) + \frac {2 \tilde {L} _ {F} ^ {2}}{S _ {1}} \\ = \frac {6 (1 - p _ {\mathrm {i n}})}{p _ {\mathrm {i n}} S _ {2}} \frac {n ^ {2}}{B _ {2} ^ {2}} \frac {L _ {F} ^ {2} \gamma^ {2}}{T} \sum_ {t = 0} ^ {T - 1} \left\| \mathbb {E} [ G ^ {t + 1} ] \right\| _ {2} ^ {2} + \frac {2 \tilde {L} _ {F} ^ {2}}{S _ {1}} + \frac {6 (1 - p _ {\mathrm {i n}})}{p _ {\mathrm {i n}} S _ {2}} \frac {n ^ {2}}{B _ {2} ^ {2}} \frac {L _ {F} ^ {2} \gamma^ {2}}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {\text {v a r}} ^ {t + 1}. \\ \end{array}
$$
Therefore, the bias has the following bound
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {\text {b i a s}} ^ {t + 1} \leq \frac {1 2 (1 - p _ {\text {i n}})}{p _ {\text {i n}} S _ {2}} \frac {n ^ {2}}{B _ {2} ^ {2}} \frac {L _ {F} ^ {2} \gamma^ {2}}{T} \sum_ {t = 0} ^ {T - 1} \| \mathbb {E} [ G ^ {t + 1} ] \| _ {2} ^ {2} + \frac {4 \tilde {L} _ {F} ^ {2}}{S _ {1}} \tag {26} \\ + \left(\frac {1 2 (1 - p _ {\mathrm {i n}})}{p _ {\mathrm {i n}} S _ {2}} \frac {n ^ {2}}{B _ {2} ^ {2}} L _ {F} ^ {2} \gamma^ {2} + \frac {2 (1 - p _ {\mathrm {o u t}}) ^ {2}}{p _ {\mathrm {o u t}}}\right) \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {\mathrm {v a r}} ^ {t + 1}. \\ \end{array}
$$
Note that when  $p_{\mathrm{in}} = 1$  and  $S_{1} = S_{2} = m$ , then this bias recovers BSpiderBoost in (16)
$$
\frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {\text {b i a s}} ^ {t + 1} \leq \frac {4 \tilde {L} _ {F} ^ {2}}{m} + \frac {2 (1 - p _ {\text {o u t}}) ^ {2}}{p _ {\text {o u t}}} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \mathcal {E} _ {\text {v a r}} ^ {t + 1}.
$$
Computing the variance. Let us decompose the variance into 3 parts:
$$
\begin{array}{l} \mathcal {E} _ {\text {v a r}} ^ {t + 1} = \mathbb {E} \left[ \left\| G ^ {t + 1} - \mathbb {E} [ G ^ {t + 1} ] \right\| _ {2} ^ {2} \right] \\ = \mathbb {E} \left[ \left\| G ^ {t + 1} \pm \hat {G} ^ {t + 1} \pm \mathbb {E} _ {\eta | i} [ \hat {G} ^ {t + 1} ] - \mathbb {E} _ {i} [ \mathbb {E} _ {\eta | i} [ \hat {G} ^ {t + 1} ] ] \right\| _ {2} ^ {2} \right] \\ = \underbrace {\mathbb {E} \left[ \left\| G ^ {t + 1} - \hat {G} ^ {t + 1} \right\| _ {2} ^ {2} \right]} _ {\mathcal {E} _ {\nabla g} ^ {t + 1}} + \underbrace {\mathbb {E} _ {i} \left[ \left\| \mathbb {E} _ {\eta | i} [ G ^ {t + 1} ] - \mathbb {E} _ {i} \left[ \mathbb {E} _ {\eta | i} [ G ^ {t + 1} ] \right] \right\| _ {2} ^ {2} \right]} _ {\mathcal {E} _ {\text {v a r , o u t}} ^ {t + 1}} + \underbrace {\mathbb {E} \left[ \left\| G ^ {t + 1} - \mathbb {E} _ {\eta | i} [ G ^ {t + 1} ] \right\| _ {2} ^ {2} \right]} _ {\mathcal {E} _ {\text {v a r , i n}} ^ {t + 1}} \\ \end{array}
$$
where  $\mathcal{E}_{\mathrm{var,out}}^{t + 1}$  and  $\mathcal{E}_{\mathrm{var,in}}^{t + 1}$  are the variance of outer loop and inner loop.
Inner Variance. For  $t \geq 1$ , we expand the inner variance