SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \left. + \left(p _ {\text {m e g a}} \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) ^ {2} + (1 - p _ {\text {m e g a}})\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \right. \\ \stackrel {(1 5)} {\leq} \frac {p _ {\mathrm {m e g a}}}{p _ {\mathrm {a}}} \mathrm {E} _ {k} \left[ \left| \left| k _ {i, 1} ^ {t + 1} - \mathrm {E} _ {k} \left[ k _ {i, 1} ^ {t + 1} \right] \right| \right| ^ {2} \right] \\ + \frac {\left(1 - p _ {\mathrm {m e g a}}\right)}{p _ {\mathrm {a}}} \mathrm {E} _ {k} \left[ \left| \left| k _ {i, 2} ^ {t + 1} - \mathrm {E} _ {k} \left[ k _ {i, 2} ^ {t + 1} \right] \right| \right| ^ {2} \right] \\ + \frac {2 (1 - p _ {\mathrm {a}})}{p _ {\mathrm {a}}} \| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) \| ^ {2} \\ + \frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {m e g a}} p _ {\mathrm {a}}} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + \left(p _ {\mathrm {m e g a}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + \left(1 - p _ {\mathrm {m e g a}}\right)\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}. \\ \end{array}
$$
Using (36) and (37), we get
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {k} \left[ \right. \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \right. \mathrm {E} _ {p _ {\mathrm {m e g a}}} \left[ \right.\left| \right.\left| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right)\right| ^ {2} \left. \right]\left. \right]\left. \right]\left. \right] \\ \leq \frac {2 b ^ {2} \sigma^ {2}}{p _ {\mathrm {a}} p _ {\mathrm {m e g a}} B ^ {\prime}} + \frac {2 p _ {\mathrm {m e g a}} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B ^ {\prime}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} \| x ^ {t + 1} - x ^ {t} \| ^ {2} \\ + \frac {\left(1 - p _ {\mathrm {m e g a}}\right) L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 (1 - p _ {\mathrm {a}})}{p _ {\mathrm {a}}} \| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) \| ^ {2} \\ + \frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {m e g a}} p _ {\mathrm {a}}} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + \left(p _ {\mathrm {m e g a}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + \left(1 - p _ {\mathrm {m e g a}}\right)\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}. \\ \end{array}
$$
Next, due to Assumption 3, we obtain
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {k} \left[ \right. \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \right. \mathrm {E} _ {p _ {\mathrm {m e g a}}} \left[ \right.\left| \right.\left| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right)\right| ^ {2} \left. \right]\left. \right]\left. \right]\left. \right] \\ \leq \frac {2 b ^ {2} \sigma^ {2}}{p _ {\mathrm {a}} p _ {\mathrm {m e g a}} B ^ {\prime}} + \left(\frac {2 p _ {\mathrm {m e g a}} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B ^ {\prime}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + \frac {(1 - p _ {\mathrm {m e g a}}) L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) L _ {i} ^ {2}}{p _ {\mathrm {a}}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \\ + \frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {m e g a}} p _ {\mathrm {a}}} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + \left(p _ {\mathrm {m e g a}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + \left(1 - p _ {\mathrm {m e g a}}\right)\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}. \\ \end{array}
$$
The third inequality can be proved with the help of (37) and Assumption 3.
$$
\begin{array}{l} \mathrm {E} _ {k} \left[ \left| \left| k _ {i, 2} ^ {t + 1} \right| \right| ^ {2} \right] \\ \stackrel {(1 6)} {=} \operatorname {E} _ {k} \left[ \left\| k _ {i, 2} ^ {t + 1} - \operatorname {E} _ {k} \left[ k _ {i, 2} ^ {t + 1} \right] \right\| ^ {2} \right] + \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \\ \leq \frac {L _ {\sigma} ^ {2}}{B} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ \leq \left(\frac {L _ {\sigma} ^ {2}}{B} + L _ {i} ^ {2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2}. \\ \end{array}
$$
$$
\operatorname {E} \left[ \left\| \nabla f (\widehat {x} ^ {T}) \right\| ^ {2} \right] \leq \frac {1}{T} \left[ \frac {2 \Delta_ {0}}{\gamma} + \frac {4}{p _ {\text {m e g a}} p _ {\mathrm {a}}} \left\| h ^ {0} - \nabla f (x ^ {0}) \right\| ^ {2} + \frac {4 \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\text {m e g a}} p _ {\mathrm {a}}} \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {0} - \nabla f _ {i} (x ^ {0}) \right\| ^ {2} \right]
$$
$$
+ \frac {1 2 \sigma^ {2}}{n B ^ {\prime}}.
$$
Proof. Due to Lemma 2 and the update step from Line 5 in Algorithm 8, we have
$$
\begin{array}{l} \mathbf {E} _ {t + 1} \left[ f (x ^ {t + 1}) \right] \\ \leq \mathrm {E} _ {t + 1} \left[ f (x ^ {t}) - \frac {\gamma}{2} \| \nabla f (x ^ {t}) \| ^ {2} - \left(\frac {1}{2 \gamma} - \frac {L}{2}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} + \frac {\gamma}{2} \| g ^ {t} - \nabla f (x ^ {t}) \| ^ {2} \right] \\ = \mathrm {E} _ {t + 1} \left[ f (x ^ {t}) - \frac {\gamma}{2} \| \nabla f (x ^ {t}) \| ^ {2} - \left(\frac {1}{2 \gamma} - \frac {L}{2}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} + \frac {\gamma}{2} \| g ^ {t} - h ^ {t} + h ^ {t} - \nabla f (x ^ {t}) \| ^ {2} \right] \\ \stackrel {(1 6)} {\leq} \operatorname {E} _ {t + 1} \left[ f (x ^ {t}) - \frac {\gamma}{2} \left\| \nabla f (x ^ {t}) \right\| ^ {2} - \left(\frac {1}{2 \gamma} - \frac {L}{2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + \gamma \left(\left\| g ^ {t} - h ^ {t} \right\| ^ {2} + \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \right). \\ \end{array}
$$
Let us fix constants  $\kappa, \eta, \nu, \rho \in [0, \infty)$  that we will define later. Considering Lemma 13, Lemma 14, and the law of total expectation, we obtain
$$
+ \frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + \left(p _ {\mathrm {m e g a}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + (1 - p _ {\mathrm {m e g a}})\right) \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2}.
$$
Let us simplify the last inequality. Since  $B' \geq B$  and  $b = \frac{p_{\mathrm{mega}} p_{\mathrm{a}}}{2 - p_{\mathrm{a}}} \leq p_{\mathrm{mega}}$ , we have  $1 - p_{\mathrm{mega}} \leq 1$ ,
$$
\frac {2 p _ {\mathrm {m e g a}} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B ^ {\prime}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} \leq \frac {2 p _ {\mathrm {m e g a}} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B},
$$
$$
\left(p _ {\text {m e g a}} \left(1 - \frac {b}{p _ {\text {m e g a}}}\right) ^ {2} + (1 - p _ {\text {m e g a}})\right) \leq 1 - b,
$$
$$
\left(\frac {2 \left(1 - p _ {\mathrm {a}}\right) b ^ {2}}{p _ {\mathrm {m e g a}} p _ {\mathrm {a}}} + p _ {\mathrm {m e g a}} \left(1 - \frac {b}{p _ {\mathrm {m e g a}}}\right) ^ {2} + (1 - p _ {\mathrm {m e g a}})\right) \leq 1 - b.
$$
$$
\begin{array}{l} \operatorname {E} \left[ f \left(x ^ {t + 1}\right) \right] + \kappa \operatorname {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \eta \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} \left[ f (x ^ {t}) - \frac {\gamma}{2} \left\| \nabla f (x ^ {t}) \right\| ^ {2} - \left(\frac {1}{2 \gamma} - \frac {L}{2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + \gamma \left(\left\| g ^ {t} - h ^ {t} \right\| ^ {2} + \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}\right) \right] \\ + \kappa \mathrm {E} \left(\frac {2 \omega}{n p _ {\mathrm {a}}} \left(\frac {L _ {\sigma} ^ {2}}{B} + \widehat {L} ^ {2}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right. \\ \left. + \frac {\left(\left(2 \omega + 1\right) p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) a ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} + (1 - a) ^ {2} \left\| g ^ {t} - h ^ {t} \right\| ^ {2}\right) \\ + \eta \mathrm {E} \left(\frac {2 \omega}{p _ {\mathrm {a}}} \left(\frac {L _ {\sigma} ^ {2}}{B} + \widehat {L} ^ {2}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right. \\ \left. + \frac {(2 \omega + 1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} + (1 - a) ^ {2} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2}\right) \\ + \nu \mathrm {E} \left(\frac {2 b ^ {2} \sigma^ {2}}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}} B ^ {\prime}} + \left(\frac {2 L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right. \\ \left. + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + (1 - b) \left\| h ^ {t} - \nabla f \left(x ^ {t}\right) \right\| ^ {2}\right) \\ + \rho \mathrm {E} \left(\frac {2 b ^ {2} \sigma^ {2}}{p _ {\mathrm {a}} p _ {\mathrm {m e g a}} B ^ {\prime}} + \left(\frac {2 L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right. \\ \left. + (1 - b) \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}\right). \\ \end{array}
$$
$$
\begin{array}{l} \operatorname {E} \left[ f \left(x ^ {t + 1}\right) \right] + \kappa \operatorname {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \eta \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \left\| \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {2 \kappa \omega}{n p _ {\mathrm {a}}} \left(\frac {L _ {\sigma} ^ {2}}{B} + \widehat {L} ^ {2}\right) - \frac {2 \eta \omega}{p _ {\mathrm {a}}} \left(\frac {L _ {\sigma} ^ {2}}{B} + \widehat {L} ^ {2}\right) \right. \\ \left. - \nu \left(\frac {2 L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) - \rho \left(\frac {2 L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \left(\gamma + \kappa (1 - a) ^ {2}\right) \operatorname {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] \\ + \left(\kappa \frac {\left((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) a ^ {2}}{n p _ {\mathrm {a}} ^ {2}} + \eta \left(\frac {(2 \omega + 1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} + (1 - a) ^ {2}\right)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + (\gamma + \nu (1 - b)) E \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\nu \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} + \rho (1 - b)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \right] \\ + \left(\frac {2 \nu b ^ {2}}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}}} + \frac {2 \rho b ^ {2}}{p _ {\mathrm {a}} p _ {\mathrm {m e g a}}}\right) \frac {\sigma^ {2}}{B ^ {\prime}}. \\ \end{array}
$$
Let us take  $\kappa = \frac{\gamma}{a}$ , thus  $\gamma + \kappa (1 - a)^2 \leq \kappa$  and
$$
\begin{array}{l} \operatorname {E} \left[ f \left(x ^ {t + 1}\right) \right] + \frac {\gamma}{a} \operatorname {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \eta \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \left| \left| \nabla f (x ^ {t}) \right| \right| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {2 \gamma \omega}{a n p _ {\mathrm {a}}} \left(\frac {L _ {\sigma} ^ {2}}{B} + \widehat {L} ^ {2}\right) - \frac {2 \eta \omega}{p _ {\mathrm {a}}} \left(\frac {L _ {\sigma} ^ {2}}{B} + \widehat {L} ^ {2}\right) \right. \\ \left. - \nu \left(\frac {2 L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) - \rho \left(\frac {2 L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{a} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] \\ \left. + \left(\frac {\gamma \left((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) a}{n p _ {\mathrm {a}} ^ {2}} + \eta \left(\frac {(2 \omega + 1 - p _ {\mathrm {a}}) a ^ {2}}{p _ {\mathrm {a}}} + (1 - a) ^ {2}\right)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \right. \\ + (\gamma + \nu (1 - b)) \operatorname {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\nu \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} + \rho (1 - b)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left| \left| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right| \right| ^ {2} \right] \\ + \left(\frac {2 \nu b ^ {2}}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}}} + \frac {2 \rho b ^ {2}}{p _ {\mathrm {a}} p _ {\mathrm {m e g a}}}\right) \frac {\sigma^ {2}}{B ^ {\prime}}. \\ \end{array}
$$
Next, since  $a = \frac{p_{\mathrm{a}}}{2\omega + 1}$ , we have  $\left(\frac{(2\omega + 1 - p_{\mathrm{a}})a^2}{p_{\mathrm{a}}} + (1 - a)^2\right) \leq 1 - a$ . We the choice  $\eta = \frac{\gamma((2\omega + 1)p_{\mathrm{a}} - p_{\mathrm{aa}})}{np_{\mathrm{a}}^2}$ , we guarantee  $\frac{\gamma((2\omega + 1)p_{\mathrm{a}} - p_{\mathrm{aa}})a}{np_{\mathrm{a}}^2} + \eta \left(\frac{(2\omega + 1 - p_{\mathrm{a}})a^2}{p_{\mathrm{a}}} + (1 - a)^2\right) \leq \eta$  and
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \end{array}
$$
where simplified the term using  $p_{\mathrm{aa}} \geq 0$ . Let us take  $\nu = \frac{\gamma}{b}$  to obtain
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma (2 \omega + 1) \omega}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {L _ {\sigma} ^ {2}}{B} + \hat {L} ^ {2}\right) \right. \\ \left. - \left(\frac {2 \gamma L _ {\sigma} ^ {2}}{b n p _ {\mathrm {a}} B} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}}\right) - \rho \left(\frac {2 L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} + \left(\frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} + \rho (1 - b)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \right] \\ + \left(\frac {2 \gamma b}{n p _ {\text {m e g a}} p _ {\mathrm {a}}} + \frac {2 \rho b ^ {2}}{p _ {\mathrm {a}} p _ {\text {m e g a}}}\right) \frac {\sigma^ {2}}{B ^ {\prime}}. \\ \end{array}
$$
Next, we take  $\rho = \frac{2\gamma(p_{\mathrm{a}} - p_{\mathrm{aa}})}{np_{\mathrm{a}}^2p_{\mathrm{mega}}}$ , thus
$$
\begin{array}{l} \operatorname {E} \left[ f \left(x ^ {t + 1}\right) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \operatorname {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma (2 \omega + 1) \omega}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {L _ {\sigma} ^ {2}}{B} + \widehat {L} ^ {2}\right) \right. \\ \left. - \left(\frac {2 \gamma L _ {\sigma} ^ {2}}{b n p _ {\mathrm {a}} B} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}}\right) - \left(\frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}}\right) \left(\frac {2 L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\frac {2 \gamma b}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}}} + \frac {4 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b ^ {2}}{n p _ {\mathrm {a}} ^ {3} p _ {\mathrm {m e g a}} ^ {2}}\right) \frac {\sigma^ {2}}{B ^ {\prime}}. \\ \end{array}
$$
Since  $\frac{p_{\mathrm{mega}}p_{\mathrm{a}}}{2} \leq b \leq p_{\mathrm{mega}}p_{\mathrm{a}}$  and  $1 - p_{\mathrm{a}} \leq 1 - \frac{p_{\mathrm{aa}}}{p_{\mathrm{a}}} \leq 1$ , we get
$$
\begin{array}{l} \mathrm {E} \left[ f \left(x ^ {t + 1}\right) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma (2 \omega + 1) \omega}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {L _ {\sigma} ^ {2}}{B} + \widehat {L} ^ {2}\right) \right. \\ \left. - \left(\frac {4 \gamma L _ {\sigma} ^ {2}}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}} ^ {2} B} + \frac {4 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}} ^ {3}}\right) - \left(\frac {4 \gamma L _ {\sigma} ^ {2}}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}} ^ {2} B} + \frac {4 \gamma (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {m e g a}} p _ {\mathrm {a}} ^ {2}}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {m e g a}}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right] \\ + \frac {6 \gamma \sigma^ {2}}{n B ^ {\prime}} \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ \end{array}
$$