SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathrm {E} _ {\mathcal {C}} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \mathrm {E} _ {p _ {\mathrm {m e g a}}} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] \right] \right] \\ \leq \frac {2 (1 - p _ {\mathrm {m e g a}}) \omega}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \left\| k _ {i, 2} ^ {t + 1} \right\| ^ {2} + \left(\frac {(p _ {\mathrm {a}} - p _ {\mathrm {a a}}) a ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} + \frac {2 (1 - p _ {\mathrm {m e g a}}) \omega a ^ {2}}{n ^ {2} p _ {\mathrm {a}}}\right) \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ + (1 - a) ^ {2} \left\| g ^ {t} - h ^ {t} \right\| ^ {2}. \\ \end{array}
$$
$$
\mathrm {E} _ {\mathcal {C}} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \mathrm {E} _ {p _ {\mathrm {m e g a}}} \left[ \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \right] \right]
$$
$$
\begin{array}{l} = p _ {\text {m e g a}} \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| g _ {i, 1} ^ {t + 1} - h _ {i, 1} ^ {t + 1} \right\| ^ {2} \right] + (1 - p _ {\text {m e g a}}) \mathrm {E} _ {\mathcal {C}} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| g _ {i, 2} ^ {t + 1} - h _ {i, 2} ^ {t + 1} \right\| ^ {2} \right] \right] \\ \stackrel {(1 6)} {=} p _ {\text {m e g a}} \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| g _ {i, 1} ^ {t + 1} - h _ {i, 1} ^ {t + 1} - (1 - a) \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right) \right\| ^ {2} \right] + (1 - p _ {\text {m e g a}}) \mathrm {E} _ {\mathcal {C}} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| g _ {i, 2} ^ {t + 1} - h _ {i, 2} ^ {t + 1} \right\| ^ {2} \right] \right] \\ + p _ {\text {m e g a}} (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ = \frac {p _ {\mathrm {m e g a}} \left(1 - p _ {\mathrm {a}}\right) a ^ {2}}{p _ {\mathrm {a}}} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} + \left(1 - p _ {\mathrm {m e g a}}\right) \mathrm {E} _ {\mathcal {C}} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| g _ {i, 2} ^ {t + 1} - h _ {i, 2} ^ {t + 1} \right\| ^ {2} \right] \right] \\ + p _ {\text {m e g a}} (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 6)} {=} \frac {p _ {\mathrm {m e g a}} (1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} + (1 - p _ {\mathrm {m e g a}}) \operatorname {E} _ {\mathcal {C}} \left[ \operatorname {E} _ {p _ {\mathrm {a}}} \left[ \| g _ {i, 2} ^ {t + 1} - h _ {i, 2} ^ {t + 1} - (1 - a) \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right) \| ^ {2} \right] \right] \\ + (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 6)} {=} \frac {p _ {\mathrm {m e g a}} (1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} + (1 - p _ {\mathrm {m e g a}}) \operatorname {E} _ {\mathcal {C}} \left[ \operatorname {E} _ {p _ {\mathrm {a}}} \left[ \| g _ {i, 2} ^ {t + 1} - h _ {i, 2} ^ {t + 1} - (1 - a) \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right) \| ^ {2} \right] \right] \\ + (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ = \frac {p _ {\mathrm {m e g a}} (1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ + \left(1 - p _ {\text {m e g a}}\right) p _ {\mathrm {a}} \mathrm {E} _ {\mathcal {C}} \left[ \left\| g _ {i} ^ {t} + \mathcal {C} _ {i} \left(\frac {1}{p _ {\mathrm {a}}} k _ {i, 2} ^ {t + 1} - \frac {a}{p _ {\mathrm {a}}} \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right)\right) - \left(h _ {i} ^ {t} + \frac {1}{p _ {\mathrm {a}}} k _ {i, 2} ^ {t + 1}\right) - (1 - a) \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right) \right\| ^ {2} \right] \\ + \left(1 - p _ {\text {m e g a}}\right) \left(1 - p _ {\mathrm {a}}\right) \left\| g _ {i} ^ {t} - h _ {i} ^ {t} - (1 - a) \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right) \right\| ^ {2} \\ + (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ = \frac {p _ {\mathrm {m e g a}} (1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ + \left(1 - p _ {\text {m e g a}}\right) p _ {\mathrm {a}} \mathrm {E} _ {\mathcal {C}} \left[ \left\| \mathcal {C} _ {i} \left(\frac {1}{p _ {\mathrm {a}}} k _ {i, 2} ^ {t + 1} - \frac {a}{p _ {\mathrm {a}}} \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right)\right) - \left(\frac {1}{p _ {\mathrm {a}}} k _ {i, 2} ^ {t + 1} - a \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right)\right) \right\| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} + \left(1 - p _ {\text {m e g a}}\right) \left(1 - p _ {\mathrm {a}}\right) a ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ + (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ \stackrel {(1 6)} {=} \left(\frac {p _ {\mathrm {m e g a}} (1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} + \frac {(1 - p _ {\mathrm {m e g a}}) (1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}}\right) \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ + \left(1 - p _ {\mathrm {m e g a}}\right) p _ {\mathrm {a}} \mathrm {E} _ {\mathcal {C}} \left[ \left\| \mathcal {C} _ {i} \left(\frac {1}{p _ {\mathrm {a}}} k _ {i, 2} ^ {t + 1} - \frac {a}{p _ {\mathrm {a}}} \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right)\right) - \left(\frac {1}{p _ {\mathrm {a}}} k _ {i, 2} ^ {t + 1} - \frac {a}{p _ {\mathrm {a}}} \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right)\right) \right\| ^ {2} \right] \\ + (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ = \frac {(1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ + \left(1 - p _ {\text {m e g a}}\right) p _ {\mathrm {a}} \mathrm {E} _ {\mathcal {C}} \left[ \left\| \mathcal {C} _ {i} \left(\frac {1}{p _ {\mathrm {a}}} k _ {i, 2} ^ {t + 1} - \frac {a}{p _ {\mathrm {a}}} \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right)\right) - \left(\frac {1}{p _ {\mathrm {a}}} k _ {i, 2} ^ {t + 1} - \frac {a}{p _ {\mathrm {a}}} \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right)\right) \right\| ^ {2} \right] \\ + (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ \leq \frac {(1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} \left| \left| g _ {i} ^ {t} - h _ {i} ^ {t} \right| \right| ^ {2} \\ + \frac {\left(1 - p _ {\mathrm {m e g a}}\right) \omega}{p _ {\mathrm {a}}} \left\| k _ {i , 2} ^ {t + 1} - a \left(g _ {i} ^ {t} - h _ {i} ^ {t}\right) \right\| ^ {2} \\ + (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \\ \stackrel {(1 5)} {\leq} \frac {2 (1 - p _ {\mathrm {m e g a}}) \omega}{p _ {\mathrm {a}}} \| k _ {i, 2} ^ {t + 1} \| ^ {2} + \left(\frac {(1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} + \frac {2 (1 - p _ {\mathrm {m e g a}}) a ^ {2} \omega}{p _ {\mathrm {a}}}\right) \| g _ {i} ^ {t} - h _ {i} ^ {t} \| ^ {2} \\ + (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2}. \\ \end{array}
$$
Lemma 14. Suppose that Assumptions 3, 5, 6 and 8 hold and let us consider sequence  $\{h_i^{t + 1}\}_{i = 1}^n$  from Algorithm 8, then
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \mathrm {E} _ {p _ {\text {m e g a}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right]\right] \\ \leq \frac {2 b ^ {2} \sigma^ {2}}{n p _ {\text {m e g a}} p _ {\mathrm {a}} B ^ {\prime}} + \left(\frac {2 p _ {\text {m e g a}} L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B ^ {\prime}} \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) ^ {2} + \frac {(1 - p _ {\text {m e g a}}) L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \\ + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2} p _ {m e g a}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + \left(p _ {m e g a} \left(1 - \frac {b}{p _ {m e g a}}\right) ^ {2} + (1 - p _ {m e g a})\right) \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}, \\ \end{array}
$$
$$
\begin{array}{l} \left. \right. \operatorname {E} _ {k} \left[ \operatorname {E} _ {p _ {\mathrm {a}}} \left[ \operatorname {E} _ {p _ {\text {m e g a}}} \left[\left\| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right)\right\| ^ {2} \right]\right]\right] \\ \leq \frac {2 b ^ {2} \sigma^ {2}}{p _ {\mathrm {a}} p _ {\text {m e g a}} B ^ {\prime}} + \left(\frac {2 p _ {\text {m e g a}} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B ^ {\prime}} \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) ^ {2} + \frac {(1 - p _ {\text {m e g a}}) L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) L _ {i} ^ {2}}{p _ {\mathrm {a}}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \\ \left. \right. + \frac {2 \left(1 - p _ {\mathrm {a}}\right) b ^ {2}}{p _ {\text {m e g a}} p _ {\mathrm {a}}} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right\| ^ {2} + \left(p _ {\text {m e g a}} \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) ^ {2} + \left(1 - p _ {\text {m e g a}}\right)\right)\left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right\| ^ {2}, \quad \forall i \in [ n ], \\ \end{array}
$$
$$
\mathrm {E} _ {k} \left[ \left\| k _ {i, 2} ^ {t + 1} \right\| ^ {2} \right] \leq \left(\frac {L _ {\sigma} ^ {2}}{B} + L _ {i} ^ {2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2}, \quad \forall i \in [ n ],
$$
Proof. First, we prove the bound for  $\operatorname{E}_k\left[\operatorname{E}_{p_{\mathrm{a}}}\left[\operatorname{E}_{p_{\mathrm{mega}}}\left[\left\|h^{t+1}-\nabla f(x^{t+1})\right\|^2\right]\right]\right]$ . Using
$$
\mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {i, 1} ^ {t + 1} \right] \right]
$$
$$
\begin{array}{l} = h _ {i} ^ {t} + \mathrm {E} _ {k} \left[ \frac {1}{B ^ {\prime}} \sum_ {k = 1} ^ {B ^ {\prime}} \nabla f _ {i} (x ^ {t + 1}; \xi_ {i k} ^ {t + 1}) - \frac {1}{B ^ {\prime}} \sum_ {k = 1} ^ {B ^ {\prime}} \nabla f _ {i} (x ^ {t}; \xi_ {i k} ^ {t + 1}) - \frac {b}{p _ {\mathrm {m e g a}}} \left(h _ {i} ^ {t} - \frac {1}{B ^ {\prime}} \sum_ {k = 1} ^ {B ^ {\prime}} \nabla f _ {i} (x ^ {t}; \xi_ {i k} ^ {t + 1})\right) \right] \\ = h _ {i} ^ {t} + \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - \frac {b}{p _ {\mathrm {m e g a}}} \left(h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t})\right) \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {i, 2} ^ {t + 1} \right] \right] \\ = h _ {i} ^ {t} + \mathrm {E} _ {k} \left[ \frac {1}{B} \sum_ {j = 1} ^ {B} \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i j} ^ {t + 1}\right) - \frac {1}{B} \sum_ {j = 1} ^ {B} \nabla f _ {i} \left(x ^ {t}; \xi_ {i j} ^ {t + 1}\right) \right] \\ = h _ {i} ^ {t} + \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right), \\ \end{array}
$$
$$
\begin{array}{l} \left. \right. \operatorname {E} _ {k} \left[ \operatorname {E} _ {p _ {\mathrm {a}}} \left[ \operatorname {E} _ {p _ {\mathrm {m e g a}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right]\right] \\ = p _ {\text {m e g a}} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {1} ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] \right] + (1 - p _ {\text {m e g a}}) \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {2} ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] \right] \\ \stackrel {(1 6)} {=} p _ {\text {m e g a}} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {1} ^ {t + 1} - \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {1} ^ {t + 1} \right] \right] \right\| ^ {2} \right] \right] + (1 - p _ {\text {m e g a}}) \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {2} ^ {t + 1} - \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {2} ^ {t + 1} \right] \right] \right\| ^ {2} \right] \right] \\ \left. + \left(p _ {\text {m e g a}} \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) ^ {2} + \left(1 - p _ {\text {m e g a}}\right)\right) \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \right. \\ \end{array}
$$
We can use Lemma 1 two times with i)  $r_i = h_i^t$  and  $s_i = k_{i,1}^{t + 1}$  and ii)  $r_i = h_i^t$  and  $s_i = k_{i,2}^{t + 1}$ , to obtain
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {k} \left[ \right. \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \right. \mathrm {E} _ {p _ {\mathrm {m e g a}}} \left[ \right.\left| \right.\left| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right| ^ {2} \left. \right]\left. \right]\left. \right]\left. \right] \\ \leq p _ {\text {m e g a}} \left(\frac {1}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {k} \left[ \left| \left| k _ {i, 1} ^ {t + 1} - \mathrm {E} _ {k} \left[ k _ {i, 1} ^ {t + 1} \right] \right| \right| ^ {2} \right] + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - \frac {b}{p _ {\text {m e g a}}} \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2}\right) \\ + \left(1 - p _ {\text {m e g a}}\right) \left(\frac {1}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {k} \left[ \left\| k _ {i, 2} ^ {t + 1} - \mathrm {E} _ {k} \left[ k _ {i, 2} ^ {t + 1} \right] \right\| ^ {2} \right] + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}\right) \\ \left. + \left(p _ {\text {m e g a}} \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) ^ {2} + (1 - p _ {\text {m e g a}})\right) \| h ^ {t} - \nabla f (x ^ {t}) \| ^ {2} \right. \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 5)} {\leq} \frac {p _ {\mathrm {m e g a}}}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \operatorname {E} _ {k} \left[ \left| \left| k _ {i, 1} ^ {t + 1} - \operatorname {E} _ {k} \left[ k _ {i, 1} ^ {t + 1} \right] \right| \right| ^ {2} \right] \\ + \frac {1 - p _ {\mathrm {m e g a}}}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {k} \left[ \left\| k _ {i, 2} ^ {t + 1} - \mathrm {E} _ {k} \left[ k _ {i, 2} ^ {t + 1} \right] \right\| ^ {2} \right] \\ + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left| \left| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) \right| \right| ^ {2} \\ + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + \left(p _ {\mathrm {m e g a}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + \left(1 - p _ {\mathrm {m e g a}}\right)\right) \left\| h ^ {t} - \nabla f \left(x ^ {t}\right) \right\| ^ {2}. \tag {35} \\ \end{array}
$$
Let us consider  $\mathrm{E}_k\left[\left\| k_{i,1}^{t + 1} - \mathrm{E}_k\left[k_{i,1}^{t + 1}\right]\right\|^2\right]$ .
$$
\begin{array}{l} \operatorname {E} _ {k} \left[ \left\| k _ {i, 1} ^ {t + 1} - \operatorname {E} _ {k} \left[ k _ {i, 1} ^ {t + 1} \right] \right\| ^ {2} \right] \\ = \mathrm {E} _ {k} \left[ \right.\left\| \frac {1}{B ^ {\prime}} \sum_ {k = 1} ^ {B ^ {\prime}} \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i k} ^ {t + 1}\right) - \frac {1}{B ^ {\prime}} \sum_ {k = 1} ^ {B ^ {\prime}} \nabla f _ {i} \left(x ^ {t}; \xi_ {i k} ^ {t + 1}\right) - \frac {b}{p _ {\text {m e g a}}} \left(h _ {i} ^ {t} - \frac {1}{B ^ {\prime}} \sum_ {k = 1} ^ {B ^ {\prime}} \nabla f _ {i} \left(x ^ {t}; \xi_ {i k} ^ {t + 1}\right)\right)\right. \\ \end{array}
$$
$$
\begin{array}{l} \left. - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - \frac {b}{p _ {\mathrm {m e g a}}} \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \| ^ {2} \right] \\ = \operatorname {E} _ {k} \left[ \right.\left\| \frac {1}{B ^ {\prime}} \sum_ {k = 1} ^ {B ^ {\prime}} \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i k} ^ {t + 1}\right) - \frac {1}{B ^ {\prime}} \sum_ {k = 1} ^ {B ^ {\prime}} \nabla f _ {i} \left(x ^ {t}; \xi_ {i k} ^ {t + 1}\right) + \frac {b}{p _ {\mathrm {m e g a}}} \left(\frac {1}{B ^ {\prime}} \sum_ {k = 1} ^ {B ^ {\prime}} \nabla f _ {i} \left(x ^ {t}; \xi_ {i k} ^ {t + 1}\right)\right)\right. \\ \left. - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) + \frac {b}{p _ {\mathrm {m e g a}}} \left(\nabla f _ {i} \left(x ^ {t}\right)\right)\right) \| ^ {2} \right] \\ = \frac {1}{B ^ {\prime 2}} \sum_ {k = 1} ^ {B ^ {\prime}} \mathrm {E} _ {k} \left[ \left\| \frac {b}{p _ {\text {m e g a}}} \left(\nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i k} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right)\right) \right. \right. \\ \left. + \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) \left(\nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i k} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i k} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \| ^ {2} \right], \\ \end{array}
$$
where we used independence of the mini-batch samples. Using (15), we get
$$
\begin{array}{l} \operatorname {E} _ {k} \left[ \left| \left| k _ {i, 1} ^ {t + 1} - \operatorname {E} _ {k} \left[ k _ {i, 1} ^ {t + 1} \right] \right| \right| ^ {2} \right] \\ \leq \frac {2 b ^ {2}}{B ^ {\prime 2} p _ {\mathrm {m e g a}} ^ {2}} \sum_ {k = 1} ^ {B ^ {\prime}} \operatorname {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i k} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \\ + \frac {2}{B ^ {\prime 2}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} \sum_ {k = 1} ^ {B ^ {\prime}} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} (x ^ {t + 1}; \xi_ {i k} ^ {t + 1}) - \nabla f _ {i} (x ^ {t}; \xi_ {i k} ^ {t + 1}) - \left(\nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t})\right) \right\| ^ {2} \right]. \\ \end{array}
$$
Due to Assumptions 5 and 6, we have
$$
\left. \right. \mathrm {E} _ {k} \left[\left\| k _ {i, 1} ^ {t + 1} - \mathrm {E} _ {k} \left[ k _ {i, 1} ^ {t + 1} \right]\right\| ^ {2} \right] \leq \frac {2 b ^ {2} \sigma^ {2}}{B ^ {\prime} p _ {\text {m e g a}} ^ {2}} + \frac {2 L _ {\sigma} ^ {2}}{B ^ {\prime}} \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) ^ {2} \| x ^ {t + 1} - x ^ {t} \| ^ {2}. \tag {36}
$$
Next, we estimate the bound for  $\mathrm{E}_k\left[\| k_{i,2}^{t + 1} - \mathrm{E}_k\left[k_{i,2}^{t + 1}\right]\| ^2\right]$ .
$$
\begin{array}{l} \mathrm {E} _ {k} \left[ \left| \left| k _ {i, 2} ^ {t + 1} - \mathrm {E} _ {k} \left[ k _ {i, 2} ^ {t + 1} \right] \right| \right| ^ {2} \right] \\ = \operatorname {E} _ {k} \left[ \left\| \frac {1}{B} \sum_ {j = 1} ^ {B} \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i j} ^ {t + 1}\right) - \frac {1}{B} \sum_ {j = 1} ^ {B} \nabla f _ {i} \left(x ^ {t}; \xi_ {i j} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \right] \\ = \frac {1}{B ^ {2}} \sum_ {j = 1} ^ {B} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i j} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i j} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \right]. \\ \end{array}
$$
Due to Assumptions 6, we have
$$
\left. \right. \mathrm {E} _ {k} \left[\left\| k _ {i, 2} ^ {t + 1} - \mathrm {E} _ {k} \left[ k _ {i, 2} ^ {t + 1} \right]\right\| ^ {2} \right] \leq \frac {L _ {\sigma} ^ {2}}{B} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2}. \tag {37}
$$
Plugging (36) and (37) into (35), we obtain
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \mathrm {E} _ {p _ {\mathrm {m e g a}}} \left[\left|\left| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right|\right| ^ {2} \right]\right]\right] \\ \leq \frac {p _ {\text {m e g a}}}{n p _ {\mathrm {a}}} \left(\frac {2 b ^ {2} \sigma^ {2}}{B ^ {\prime} p _ {\text {m e g a}} ^ {2}} + \frac {2 L _ {\sigma} ^ {2}}{B ^ {\prime}} \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) ^ {2} \| x ^ {t + 1} - x ^ {t} \| ^ {2}\right) \\ + \frac {\left(1 - p _ {\mathrm {m e g a}}\right) L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ \end{array}
$$
$$
+ \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + \left(p _ {\mathrm {m e g a}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + (1 - p _ {\mathrm {m e g a}})\right) \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}.
$$
Using Assumption 3, we get
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \mathrm {E} _ {p _ {\mathrm {m e g a}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right]\right] \\ \leq \frac {2 b ^ {2} \sigma^ {2}}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}} B ^ {\prime}} + \left(\frac {2 p _ {\mathrm {m e g a}} L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B ^ {\prime}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + \frac {(1 - p _ {\mathrm {m e g a}}) L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \\ + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + \left(p _ {\mathrm {m e g a}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + (1 - p _ {\mathrm {m e g a}})\right) \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
Using almost the same derivations, we can prove the second inequality:
$$
= p _ {\text {m e g a}} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i, 1} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \right] + (1 - p _ {\text {m e g a}}) \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i, 2} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \right]
$$
$$
\stackrel {(1 6)} {=} p _ {\mathrm {m e g a}} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i, 1} ^ {t + 1} - \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {i, 1} ^ {t + 1} \right] \right] \right\| ^ {2} \right] \right] + (1 - p _ {\mathrm {m e g a}}) \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i, 2} ^ {t + 1} - \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {i, 2} ^ {t + 1} \right] \right] \right\| ^ {2} \right] \right]
$$