SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i j} ^ {t + 1} - \nabla f _ {i j} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \right] \\ \leq \frac {2 \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) L _ {\max } ^ {2}}{\frac {p _ {\mathrm {a}} B}{m}} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \left(\frac {2 \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) b ^ {2}}{\frac {p _ {\mathrm {a}} B}{m}} + (1 - b) ^ {2}\right) \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2}, \quad \forall i \in [ n ], \forall j \in [ m ]. \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \left\| k _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ \leq \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 L _ {i} ^ {2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 b ^ {2}}{B m} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} + 2 b ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}, \quad \forall i \in [ n ]. \\ \end{array}
$$
Proof. We start by proving the first inequality. Note that
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {i} ^ {t + 1} \right] \right] \\ = p _ {\mathrm {a}} \left(h _ {i} ^ {t} + \frac {1}{p _ {\mathrm {a}}} \mathrm {E} _ {B} \left[ h _ {i} ^ {t + 1} \right]\right) + (1 - p _ {\mathrm {a}}) h _ {i} ^ {t} \\ = h _ {i} ^ {t} + \frac {1}{m} \sum_ {j = 1} ^ {m} \frac {B}{m} \cdot \frac {m}{B} \left(\nabla f _ {i j} \left(x ^ {t + 1}\right) - \nabla f _ {i j} \left(x ^ {t}\right) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right)\right)\right) + \left(1 - \frac {B}{m}\right) \cdot 0 \\ = \nabla f _ {i} \left(x ^ {t + 1}\right) + (1 - b) \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right), \\ \end{array}
$$
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right] \\ \stackrel {(1 6)} {=} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h ^ {t + 1} - \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} [ h ^ {t + 1} ] \right] \right\| ^ {2} \right] \right] + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
We can use Lemma 1 with  $r_i = h_i^t$  and  $s_i = k_i^{t+1}$  to obtain
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right] \\ \leq \frac {1}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {B} \left[ \left\| k _ {i} ^ {t + 1} - \mathrm {E} _ {B} \left[ k _ {i} ^ {t + 1} \right] \right\| ^ {2} \right] + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \mathrm {E} _ {B} \left[ k _ {i} ^ {t + 1} \right] \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \\ = \frac {1}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {B} \left[ \left\| \frac {1}{m} \sum_ {j = 1} ^ {m} k _ {i j} ^ {t + 1} - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
Next, we again use Lemma 1 with  $r_i = 0$ ,  $s_i = \nabla f_{ij}(x^{t + 1}) - \nabla f_{ij}(x^t) - b\left(h_{ij}^t -\nabla f_{ij}(x^t)\right)$ ,  $p_{\mathrm{a}} = \frac{B}{m}$ , and  $p_{\mathrm{aa}} = \frac{B(B - 1)}{m(m - 1)}$ :
$$
\left. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] \right] \right.
$$
$$
\begin{array}{l} \leq \frac {1}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \left(\frac {m - B}{B m (m - 1)} \sum_ {j = 1} ^ {m} \left\| \nabla f _ {i j} (x ^ {t + 1}) - \nabla f _ {i j} (x ^ {t}) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t})\right) \right\| ^ {2}\right) \\ + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - b \left(h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t})\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \\ \leq \frac {1}{n ^ {2} p _ {\mathrm {a}} B m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| \nabla f _ {i j} \left(x ^ {t + 1}\right) - \nabla f _ {i j} \left(x ^ {t}\right) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - b \left(h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t})\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 5)} {\leq} \frac {2}{n ^ {2} p _ {\mathrm {a}} B m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| \nabla f _ {i j} (x ^ {t + 1}) - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} + \frac {2 b ^ {2}}{n ^ {2} p _ {\mathrm {a}} B m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} \\ + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
Due to Assumptions 3 and 4, we have
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[\left|\left| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right|\right| ^ {2} \right]\right] \\ \leq \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + \frac {2 b ^ {2}}{n ^ {2} p _ {\mathrm {a}} B m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
Let us get the bound for the second inequality:
$$
\begin{array}{l} \left. \right. \operatorname {E} _ {B} \left[ \operatorname {E} _ {p _ {\mathrm {a}}} \left[\left\| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right)\right\| ^ {2} \right]\right] \\ \stackrel {(1 6)} {=} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i} ^ {t + 1} - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) + (1 - b) \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \right] \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ = p _ {\mathrm {a}} \mathrm {E} _ {B} \left[ \left\| h _ {i} ^ {t} + \frac {1}{p _ {\mathrm {a}}} k _ {i} ^ {t + 1} - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) + (1 - b) \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ \left. + \left(1 - p _ {\mathrm {a}}\right) \left\| h _ {i} ^ {t} - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) + (1 - b) \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right. \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 6)} {=} \frac {1}{p _ {\mathrm {a}}} \operatorname {E} _ {B} \left[ \left\| k _ {i} ^ {t + 1} - \operatorname {E} _ {B} \left[ k _ {i} ^ {t + 1} \right] \right\| ^ {2} \right] \\ + \frac {1 - p _ {\mathrm {a}}}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}. \\ \end{array}
$$
Let us use Lemma 1 with  $r_i = 0$ ,  $s_i = \nabla f_{ij}(x^{t + 1}) - \nabla f_{ij}(x^t) - b(h_{ij}^t -\nabla f_{ij}(x^t))$ ,  $p_{\mathrm{a}} = \frac{B}{m}$ , and  $p_{\mathrm{aa}} = \frac{B(B - 1)}{m(m - 1)}$ :
$$
\left. \right. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[\left|\left| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right)\right|\right| ^ {2} \right]\right]
$$
$$
\begin{array}{l} \leq \frac {1}{p _ {\mathrm {a}}} \left(\frac {m - B}{B m (m - 1)} \sum_ {j = 1} ^ {m} \left\| \nabla f _ {i j} \left(x ^ {t + 1}\right) - \nabla f _ {i j} \left(x ^ {t}\right) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right)\right) \right\| ^ {2}\right) \\ + \frac {1 - p _ {\mathrm {a}}}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \\ \leq \frac {1}{p _ {\mathrm {a}} B m} \sum_ {j = 1} ^ {m} \left\| \nabla f _ {i j} (x ^ {t + 1}) - \nabla f _ {i j} (x ^ {t}) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t})\right) \right\| ^ {2} \\ + \frac {1 - p _ {\mathrm {a}}}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \\ \stackrel {(1 5)} {\leq} \frac {2}{p _ {\mathrm {a}} B m} \sum_ {j = 1} ^ {m} \left\| \nabla f _ {i j} \left(x ^ {t + 1}\right) - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} + \frac {2 \left(1 - p _ {\mathrm {a}}\right)}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ + \frac {2 b ^ {2}}{p _ {\mathrm {a}} B m} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} + \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \\ \leq \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) L _ {i} ^ {2}}{p _ {\mathrm {a}}}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 b ^ {2}}{p _ {\mathrm {a}} B m} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} + \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2}, \\ \end{array}
$$
where we used Assumptions 3 and 4. We continue the proof by considering
$$
\mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i j} ^ {t + 1} - \nabla f _ {i j} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \right]:
$$
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[\left\| h _ {i j} ^ {t + 1} - \nabla f _ {i j} \left(x ^ {t + 1}\right)\right\| ^ {2} \right]\right] \\ \stackrel {(1 6)} {=} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i j} ^ {t + 1} - \left(\nabla f _ {i j} \left(x ^ {t + 1}\right) + (1 - b) \left(h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \right] \\ + (1 - b) ^ {2} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} \\ = \frac {p _ {\mathrm {a}} B}{m} \mathrm {E} _ {B} \left[ \left\| h _ {i j} ^ {t} + \frac {m}{B p _ {\mathrm {a}}} \left(\nabla f _ {i j} (x ^ {t + 1}) - \nabla f _ {i j} (x ^ {t}) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t})\right)\right) - \left(\nabla f _ {i j} (x ^ {t + 1}) + (1 - b) (h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}))\right) \right\| ^ {2} \right] \\ \left. + \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) \left\| h _ {i j} ^ {t} - \left(\nabla f _ {i j} \left(x ^ {t + 1}\right) + (1 - b) \left(h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right. \\ + (1 - b) ^ {2} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} \\ = \frac {\left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) ^ {2}}{\frac {p _ {\mathrm {a}} B}{m}} \left\| \nabla f _ {i j} \left(x ^ {t + 1}\right) - \nabla f _ {i j} \left(x ^ {t}\right) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) \left\| \nabla f _ {i j} \left(x ^ {t + 1}\right) - \nabla f _ {i j} \left(x ^ {t}\right) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} \\ = \frac {\left(1 - \frac {p _ {\mathrm {a}} B}{m}\right)}{\frac {p _ {\mathrm {a}} B}{m}} \left\| \nabla f _ {i j} (x ^ {t + 1}) - \nabla f _ {i j} (x ^ {t}) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t})\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} \\ \stackrel {(1 5)} {\leq} \frac {2 \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right)}{\frac {p _ {\mathrm {a}} B}{m}} \left\| \nabla f _ {i j} (x ^ {t + 1}) - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} + \left(\frac {2 \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) b ^ {2}}{\frac {p _ {\mathrm {a}} B}{m}} + (1 - b) ^ {2}\right) \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
It is left to consider Assumption 4:
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i j} ^ {t + 1} - \nabla f _ {i j} (x ^ {t + 1}) \right\| ^ {2} \right] \right] \\ \leq \frac {2 \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) L _ {\max} ^ {2}}{\frac {p _ {\mathrm {a}} B}{m}} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + \left(\frac {2 \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) b ^ {2}}{\frac {p _ {\mathrm {a}} B}{m}} + (1 - b) ^ {2}\right) \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
Finally, we obtain the bound for the last inequality of the lemma:
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \left| \left| k _ {i} ^ {t + 1} \right| \right| ^ {2} \right] \\ \stackrel {(1 6)} {=} \operatorname {E} _ {B} \left[ \left\| k _ {i} ^ {t + 1} - \operatorname {E} _ {B} \left[ k _ {i} ^ {t + 1} \right] \right\| ^ {2} \right] \\ + \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2}. \\ \end{array}
$$
Using Lemma 1, we get
$$
\begin{array}{l} \operatorname {E} _ {B} \left[ \left| \left| k _ {i} ^ {t + 1} \right| \right| ^ {2} \right] \\ \leq \frac {m - B}{B m (m - 1)} \sum_ {j = 1} ^ {m} \left\| \nabla f _ {i j} \left(x ^ {t + 1}\right) - \nabla f _ {i j} \left(x ^ {t}\right) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ \leq \frac {1}{B m} \sum_ {j = 1} ^ {m} \left\| \nabla f _ {i j} \left(x ^ {t + 1}\right) - \nabla f _ {i j} \left(x ^ {t}\right) - b \left(h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - b (h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t})) \right\| ^ {2} \\ \stackrel {(1 5)} {\leq} \frac {2}{B m} \sum_ {j = 1} ^ {m} \left\| \nabla f _ {i j} (x ^ {t + 1}) - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} + 2 \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \\ + \frac {2 b ^ {2}}{B m} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} + 2 b ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \\ \leq \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 L _ {i} ^ {2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 b ^ {2}}{B m} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} + 2 b ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2}, \\ \end{array}
$$
where we used Assumptions 3 and 4.
Theorem 7. Suppose that Assumptions 1, 2, 3, 4, 7, and 8 hold. Let us take  $a = \frac{p_{\mathrm{a}}}{2\omega + 1}$ ,  $b = \frac{\frac{p_{\mathrm{a}}B}{m}}{2 - \frac{p_{\mathrm{a}}B}{m}}$ ,
$$
\gamma \leq \left(L + \sqrt {\frac {1 4 8 \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\widehat {L} ^ {2} + \frac {L _ {\operatorname* {m a x}} ^ {2}}{B}\right) + \frac {7 2 m}{n p _ {\mathrm {a}} ^ {2} B} \left(\left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {L _ {\operatorname* {m a x}} ^ {2}}{B}\right)}\right) ^ {- 1},
$$
$g_{i}^{0} = h_{i}^{0} = \nabla f_{i}(x^{0})$  for all  $i\in [n]$  and  $h_{ij}^{0} = \nabla f_{ij}(x^0)$  for all  $i\in [n],j\in [m]$  in Algorithm 1 (DASHA-PP-FINITE-MVR) then  $\operatorname {E}\left[\left\| \nabla f(\hat{x}^T)\right\| ^2\right]\leq \frac{2\Delta_0}{\gamma T}$
Proof. Let us fix constants  $\nu, \rho, \delta \in [0, \infty)$  that we will define later. Considering Lemma 6, Lemma 9, and the law of total expectation, we obtain
$$
\begin{array}{l} + \rho \mathrm {E} \left(\left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right. \\ \left. + \frac {2 b ^ {2}}{p _ {\mathrm {a}} B n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} + \left(\frac {2 \left(1 - p _ {\mathrm {a}}\right) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}\right) \\ + \delta \mathrm {E} \left(\frac {2 \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) L _ {\max } ^ {2}}{\frac {p _ {\mathrm {a}} B}{m}} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \right. \\ \left. + \left(\frac {2 \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) b ^ {2}}{\frac {p _ {\mathrm {a}} B}{m}} + (1 - b) ^ {2}\right) \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2}\right). \\ \end{array}
$$
Due to  $b = \frac{\frac{p_{\mathrm{a}}B}{m}}{2 - \frac{p_{\mathrm{a}}B}{m}} \leq \frac{p_{\mathrm{a}}}{2 - p_{\mathrm{a}}}$ , we have
$$
\left(\frac {2 \left(1 - \frac {p _ {\mathrm {a}} B}{m}\right) b ^ {2}}{\frac {p _ {\mathrm {a}} B}{m}} + (1 - b) ^ {2}\right) \leq 1 - b
$$
$$
\left(\frac {2 \left(1 - p _ {\mathrm {a}}\right) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \leq 1 - b.
$$
Moreover, we consider that  $1 - \frac{p_{\mathrm{a}}B}{m}\leq 1$  , therefore
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ + \delta \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t + 1} - \nabla f _ {i j} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \\ \leq \operatorname {E} \left[ f (x ^ {t}) - \frac {\gamma}{2} \| \nabla f (x ^ {t}) \| ^ {2} - \left(\frac {1}{2 \gamma} - \frac {L}{2}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} + \gamma \| h ^ {t} - \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \left(\frac {2 L _ {\mathrm {m a x}} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + \frac {2 b ^ {2}}{B m n} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} + \frac {2 b ^ {2}}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right] \\ + \nu \mathrm {E} \left(\left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right. \\ + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + \frac {2 b ^ {2}}{n ^ {2} p _ {\mathrm {a}} B m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} \\ \left. + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f \left(x ^ {t}\right) \right\| ^ {2}\right) \\ + \rho \mathrm {E} \left(\left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right. \\ \end{array}
$$
$$
\begin{array}{l} \left. + \frac {2 b ^ {2}}{p _ {\mathrm {a}} B n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} + (1 - b) \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}\right) \\ + \delta \mathrm {E} \left(\frac {2 m L _ {\operatorname* {m a x}} ^ {2}}{p _ {\mathrm {a}} B} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + (1 - b) \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2}\right). \\ \end{array}
$$