SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} = \frac {2 b ^ {2}}{n ^ {2} p _ {\mathrm {a}} B ^ {2}} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {B} \operatorname {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i j} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \\ + \frac {2 (1 - b) ^ {2}}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \right] \\ + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - b (h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t})) \right\| ^ {2} \\ + \left(1 - b\right) ^ {2} \left| \left| h ^ {t} - \nabla f (x ^ {t}) \right| \right| ^ {2}. \\ \end{array}
$$
In the last equality, we use the independence of elements in the mini-batches. Due to Assumption 5, we get
$$
\left. \right. \operatorname {E} _ {k} \left[ \operatorname {E} _ {p _ {a}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right]
$$
$$
\begin{array}{l} \leq \frac {2 b ^ {2} \sigma^ {2}}{n p _ {\mathrm {a}} B} \\ + \frac {2 (1 - b) ^ {2}}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} (x ^ {t + 1}; \xi_ {i} ^ {t + 1}) - \nabla f _ {i} (x ^ {t}; \xi_ {i} ^ {t + 1}) - \left(\nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t})\right) \right\| ^ {2} \right] \\ + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - b (h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t})) \right\| ^ {2} \\ + \left(1 - b\right) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 5)} {\leq} \frac {2 b ^ {2} \sigma^ {2}}{n p _ {\mathrm {a}} B} \\ + \frac {2 (1 - b) ^ {2}}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} (x ^ {t + 1}; \xi_ {i} ^ {t + 1}) - \nabla f _ {i} (x ^ {t}; \xi_ {i} ^ {t + 1}) - \left(\nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t})\right) \right\| ^ {2} \right] \\ + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ = \frac {2 b ^ {2} \sigma^ {2}}{n p _ {\mathrm {a}} B} \\ + \frac {2 (1 - b) ^ {2}}{n ^ {2} p _ {\mathrm {a}} B ^ {2}} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {B} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i j} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i j} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \right] \\ + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}, \\ \end{array}
$$
where we use the independence of elements in the mini-batches. Using Assumptions 3 and 6, we obtain
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right] \\ \leq \frac {2 b ^ {2} \sigma^ {2}}{n p _ {\mathrm {a}} B} + \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
Now, we prove the second inequality:
Considering the independence of elements in the mini-batch, we obtain
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[\left\| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right)\right\| ^ {2} \right]\right] \\ = \frac {2 b ^ {2}}{p _ {\mathrm {a}} B ^ {2}} \sum_ {j = 1} ^ {B} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i j} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \\ + \frac {2 (1 - b) ^ {2}}{p _ {\mathrm {a}} B ^ {2}} \sum_ {j = 1} ^ {B} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i j} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i j} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \right] \\ + \frac {1 - p _ {\mathrm {a}}}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 5)} {\leq} \frac {2 b ^ {2}}{p _ {\mathrm {a}} B ^ {2}} \sum_ {j = 1} ^ {B} \operatorname {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i j} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \\ + \frac {2 (1 - b) ^ {2}}{p _ {\mathrm {a}} B ^ {2}} \sum_ {j = 1} ^ {B} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i j} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i j} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \right] \\ + \frac {2 (1 - p _ {\mathrm {a}})}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ \end{array}
$$
Next, we use Assumptions 3, 6, 5, to get
$$
\begin{array}{l} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left| \left| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right| \right| ^ {2} \right] \right] \\ \leq \frac {2 b ^ {2} \sigma^ {2}}{p _ {\mathrm {a}} B} + \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) L _ {i} ^ {2}}{p _ {\mathrm {a}}}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ \left. + \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}. \right. \\ \end{array}
$$
It is left to prove the bound for  $\mathrm{E}_k\left[\left\| k_i^{t + 1}\right\| ^2\right]$
$$
\begin{array}{l} \mathrm {E} _ {k} \left[ \left\| k _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ = \operatorname {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right)\right) \right\| ^ {2} \right] \\ \stackrel {(1 6)} {=} \operatorname {E} _ {k} \left[ \left\| \nabla f _ {i} (x ^ {t + 1}; \xi_ {i} ^ {t + 1}) - \nabla f _ {i} (x ^ {t}; \xi_ {i} ^ {t + 1}) - b \left(h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}; \xi_ {i} ^ {t + 1})\right) - \left(\nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - b (h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}))\right) \right\| ^ {2} \right] \\ + \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ = \operatorname {E} _ {k} \left[ \left\| b \left(\nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right)\right) + (1 - b) \left(\nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ + \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ \stackrel {(1 5)} {\leq} 2 b ^ {2} \mathrm {E} _ {k} \left[ \left| \left| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right) \right| \right| ^ {2} \right] \\ + 2 (1 - b) ^ {2} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} (x ^ {t + 1}; \xi_ {i} ^ {t + 1}) - \nabla f _ {i} (x ^ {t}; \xi_ {i} ^ {t + 1}) - \left(\nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t})\right) \right\| ^ {2} \right] \\ + 2 \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + 2 b ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}. \\ \end{array}
$$
Using Assumptions 3, 6, 5 and the independence of elements in the mini-batch, we get
$$
\operatorname {E} _ {k} \left[ \left| \left| k _ {i} ^ {t + 1} \right| \right| ^ {2} \right]
$$
$$
\leq \frac {2 b ^ {2} \sigma^ {2}}{B} + \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 L _ {i} ^ {2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + 2 b ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2}.
$$
Theorem 4. Suppose that Assumptions 1, 2, 3, 5, 6, 7 and 8 hold. Let us take  $a = \frac{p_{\mathrm{a}}}{2\omega + 1}$ ,
$b\in \left(0,\frac{p_{\mathrm{a}}}{2 - p_{\mathrm{a}}}\right],\gamma \leq \left(L + \left[\frac{48\omega(2\omega + 1)}{np_{\mathrm{a}}^{2}}\left(\widehat{L}^{2} + \frac{(1 - b)^{2}L_{\sigma}^{2}}{B}\right) + \frac{12}{np_{\mathrm{a}}b}\left(\left(1 - \frac{p_{\mathrm{aa}}}{p_{\mathrm{a}}}\right)\widehat{L}^{2} + \frac{(1 - b)^{2}L_{\sigma}^{2}}{B}\right)\right]^{1 / 2}\right)^{-1},$  and  $g_i^0 = h_i^0$  for all  $i\in [n]$  in Algorithm 1 (DASHA-PP-MVR). Then
$$
\begin{array}{l} \operatorname {E} \left[ \left\| \nabla f (\widehat {x} ^ {T}) \right\| ^ {2} \right] \leq \frac {1}{T} \left[ \frac {2 \Delta_ {0}}{\gamma} + \frac {2}{b} \left\| h ^ {0} - \nabla f (x ^ {0}) \right\| ^ {2} + \left(\frac {3 2 b \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {4 \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\mathrm {a}}}\right) \left(\frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {0} - \nabla f _ {i} (x ^ {0}) \right\| ^ {2}\right) \right] \\ + \left(\frac {4 8 b ^ {2} \omega (2 \omega + 1)}{p _ {\mathrm {a}} ^ {2}} + \frac {1 2 b}{p _ {\mathrm {a}}}\right) \frac {\sigma^ {2}}{n B}. \\ \end{array}
$$
Proof. Let us fix constants  $\nu, \rho \in [0, \infty)$  that we will define later. Considering Lemma 6, Lemma 10, and the law of total expectation, we obtain
$$
\begin{array}{l} + \nu \mathrm {E} \left(\frac {2 b ^ {2} \sigma^ {2}}{n p _ {\mathrm {a}} B} + \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right. \\ \left. + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f \left(x ^ {t}\right) \right\| ^ {2}\right) \\ + \rho \mathrm {E} \left(\frac {2 b ^ {2} \sigma^ {2}}{p _ {\mathrm {a}} B} + \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right. \\ \left. + \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \frac {1}{n} \sum_ {i = 1} ^ {n} \left| \left| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right| \right| ^ {2}\right). \\ \end{array}
$$
$$
\begin{array}{l} \operatorname {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \operatorname {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \left| \left| \nabla f (x ^ {t}) \right| \right| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \right. \\ \left. - \nu \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) - \rho \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \left(\gamma + \nu (1 - b) ^ {2}\right) \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ \left. \right. + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \nu \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n p _ {\mathrm {a}} ^ {2}} + \rho \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right\| ^ {2} \right] \\ + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \nu \frac {2 b ^ {2}}{n p _ {\mathrm {a}}} + \rho \frac {2 b ^ {2}}{p _ {\mathrm {a}}}\right) \frac {\sigma^ {2}}{B}. \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} \left[ f \left(x ^ {t + 1}\right) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t} - h ^ {t} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t} - h _ {i} ^ {t} \| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \right. \\ \left. - \frac {\gamma}{b} \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) - \rho \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} \left. \right. + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b}{n p _ {\mathrm {a}} ^ {2}} + \rho \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma b}{n p _ {\mathrm {a}}} + \rho \frac {2 b ^ {2}}{p _ {\mathrm {a}}}\right) \frac {\sigma^ {2}}{B}. \\ \end{array}
$$
Note that  $b \leq \frac{p_{\mathrm{a}}}{2 - p_{\mathrm{a}}}$ , thus
$$
\begin{array}{l} \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b}{n p _ {\mathrm {a}} ^ {2}} + \rho \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right)\right) \\ \leq \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b}{n p _ {\mathrm {a}} ^ {2}} + \rho (1 - b)\right). \\ \end{array}
$$
$$
\left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b}{n p _ {\mathrm {a}} ^ {2}} + \rho (1 - b)\right) \leq \rho ,
$$
$$
\begin{array}{l} \operatorname {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \operatorname {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \right. \\ - \frac {\gamma}{n p _ {\mathrm {a}} b} \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2}\right) \\ \left. - \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {3}} + \frac {2 \gamma \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\mathrm {a}} ^ {2}}\right) \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma b}{n p _ {\mathrm {a}}} + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \frac {2 b ^ {2}}{p _ {\mathrm {a}}}\right) \frac {\sigma^ {2}}{B}. \\ \end{array}
$$
Let us simplify the inequality. First, due to  $b \leq p_{\mathrm{a}}$  and  $(1 - p_{\mathrm{a}}) \leq \left(1 - \frac{p_{\mathrm{aa}}}{p_{\mathrm{a}}} \right)$ , we have
$$
\begin{array}{l} \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {3}} + \frac {2 \gamma \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\mathrm {a}} ^ {2}}\right) \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}\right) \\ = \frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {3}} \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}\right) \\ + \frac {2 \gamma \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}\right) \\ \end{array}
$$
$$
\begin{array}{l} \leq \frac {8 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \\ + \frac {2 \gamma}{n p _ {\mathrm {a}} b} \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2}\right), \\ \end{array}
$$
Also, we can simplify the last term:
$$
\begin{array}{l} \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \frac {2 b ^ {2}}{p _ {\mathrm {a}}} \\ = \frac {1 6 b ^ {3} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {3}} + \frac {4 b ^ {2} \gamma \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\mathrm {a}} ^ {2}} \\ \leq \frac {1 6 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {4 b \gamma}{n p _ {\mathrm {a}}}, \\ \end{array}
$$