SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ = \operatorname {E} \left[ f \left(x ^ {t + 1}\right) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \frac {\gamma \left(\left(2 \omega + 1\right) p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ \left. \right. + \nu \mathrm {E} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right] + \rho \mathrm {E} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right]\right] \\ \leq \operatorname {E} \left[ f (x ^ {t}) - \frac {\gamma}{2} \| \nabla f (x ^ {t}) \| ^ {2} - \left(\frac {1}{2 \gamma} - \frac {L}{2}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} + \gamma \| h ^ {t} - \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ 2 \widehat {L} ^ {2} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + 2 b ^ {2} \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + \frac {2 b ^ {2} \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f \left(x ^ {t}\right) \right\| ^ {2} \right] \\ + \rho \mathrm {E} \left[ \frac {2 (1 - p _ {\mathrm {a}})}{p _ {\mathrm {a}}} \widehat {L} ^ {2} \| x ^ {t + 1} - x ^ {t} \| ^ {2} + \left(\frac {2 b ^ {2} (1 - p _ {\mathrm {a}})}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \frac {1}{n} \sum_ {i = 1} ^ {n} \| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \| ^ {2} \right]. \\ \end{array}
$$
After rearranging the terms, we get
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {8 \gamma \omega (2 \omega + 1) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} - \nu \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} - \rho \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) \operatorname {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \left(\gamma + \nu (1 - b) ^ {2}\right) \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ \left. \right. + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \nu \frac {2 b ^ {2} \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}} + \rho \left(\frac {2 b ^ {2} (1 - p _ {\mathrm {a}})}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right\| ^ {2} \right]. \\ \end{array}
$$
By taking  $\nu = \frac{\gamma}{b}$ , one can show that  $(\gamma + \nu (1 - b)^2) \leq \nu$ , and
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {8 \gamma \omega (2 \omega + 1) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} - \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}} - \rho \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) \operatorname {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ \left. \right. + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma b \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}} + \rho \left(\frac {2 b ^ {2} (1 - p _ {\mathrm {a}})}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
Note that  $b = \frac{p_{\mathrm{a}}}{2 - p_{\mathrm{a}}}$ , thus
$$
\begin{array}{l} \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma b \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}} + \rho \left(\frac {2 b ^ {2} (1 - p _ {\mathrm {a}})}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right)\right) \\ \leq \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma b \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}} + \rho (1 - b)\right). \\ \end{array}
$$
And if we take  $\rho = \frac{8b\gamma\omega(2\omega + 1)}{np_{\mathrm{a}}^2} +\frac{2\gamma(p_{\mathrm{a}} - p_{\mathrm{aa}})}{np_{\mathrm{a}}^2}$ , then
$$
\left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma b \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}} + \rho (1 - b)\right) \leq \rho ,
$$
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {8 \gamma \omega (2 \omega + 1) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} - \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}} \right. \\ \left. - \frac {1 6 b \gamma \omega (2 \omega + 1) (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {3}} - \frac {4 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {3}}\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
Let us simplify the last inequality. First, note that
$$
\frac {1 6 b \gamma \omega (2 \omega + 1) (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {3}} \leq \frac {1 6 \gamma \omega (2 \omega + 1) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}},
$$
due to  $b \leq p_{\mathrm{a}}$ . Second,
$$
\frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}} \leq \frac {4 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {3}},
$$
due to  $b \geq \frac{p_{\mathrm{a}}}{2}$ . All in all, we have
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \left| \left| \nabla f (x ^ {t}) \right| \right| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {2 4 \gamma \omega (2 \omega + 1) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} - \frac {8 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {3}}\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
Using Lemma 4 and the assumption about  $\gamma$ , we get
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
It is left to apply Lemma 3 with
$$
\begin{array}{l} \Psi^ {t} = \frac {(2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {1}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \left(\frac {8 b \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right] \\ \end{array}
$$
to conclude the proof.
E.4 Proof for DASHA-PP-PAGE
Let us denote
$$
k _ {i, 1} ^ {t + 1} := \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - \frac {b}{p _ {\text {p a g e}}} \left(h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t})\right),
$$
$$
k _ {i, 2} ^ {t + 1} := \frac {1}{B} \sum_ {j \in I _ {i} ^ {t}} \left(\nabla f _ {i j} (x ^ {t + 1}) - \nabla f _ {i j} (x ^ {t})\right),
$$
$$
h _ {i, 1} ^ {t + 1} := \left\{ \begin{array}{l l} h _ {i} ^ {t} + \frac {1}{p _ {\mathrm {a}}} k _ {i, 1} ^ {t + 1}, & i ^ {\text {t h}} \text {n o d e i s p a r t i c i p a t i n g}, \\ h _ {i} ^ {t}, & \text {o t h e r w i s e}, \end{array} \right.
$$
$$
h _ {i, 2} ^ {t + 1} := \left\{ \begin{array}{l l} h _ {i} ^ {t} + \frac {1}{p _ {\mathrm {a}}} k _ {i, 2} ^ {t + 1}, & i ^ {\text {t h}} \text {n o d e i s p a r t i c i p a t i n g}, \\ h _ {i} ^ {t}, & \text {o t h e r w i s e}, \end{array} \right.
$$
$$
h _ {1} ^ {t + 1} := \frac {1}{n} \sum_ {i = 1} ^ {n} h _ {i, 1} ^ {t + 1}, \text {a n d} h _ {2} ^ {t + 1} := \frac {1}{n} \sum_ {i = 1} ^ {n} h _ {i, 2} ^ {t + 1}. \text {N o t e , t h a t}
$$
$$
h ^ {t + 1} = \left\{ \begin{array}{l l} h _ {1} ^ {t + 1}, & \text {w i t h p r o b a b i l i t y} p _ {\text {p a g e}}, \\ h _ {2} ^ {t + 1}, & \text {w i t h p r o b a b i l i t y} 1 - p _ {\text {p a g e}}. \end{array} \right.
$$
Lemma 8. Suppose that Assumptions 3, 4, and 8 hold. For  $h_i^{t+1}$  and  $k_i^{t+1}$  from Algorithm 1 (DASHA-PP-PAGE) we have
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \mathrm {E} _ {p _ {\text {p a g e}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right]\right] \\ \leq \left(\frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} + \frac {(1 - p _ {p a g e}) L _ {\max} ^ {2}}{n p _ {\mathrm {a}} B}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2} p _ {p a g e}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + \left(p _ {p a g e} \left(1 - \frac {b}{p _ {p a g e}}\right) ^ {2} + (1 - p _ {p a g e})\right) \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \mathrm {E} _ {p _ {p a g e}} \left[\left\| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right)\right\| ^ {2} \right]\right]\right] \\ \leq \left(\frac {2 (1 - p _ {\mathrm {a}}) L _ {i} ^ {2}}{p _ {\mathrm {a}}} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{p _ {\mathrm {a}} B}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \\ + \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}} p _ {p a g e}} + p _ {p a g e} \left(1 - \frac {b}{p _ {p a g e}}\right) ^ {2} + (1 - p _ {p a g e})\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2}, \quad \forall i \in [ n ]. \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\text {p a g e}}} \left[ \left\| k _ {i} ^ {t + 1} \right\| ^ {2} \right] \right] \\ \leq \left(2 L _ {i} ^ {2} + \frac {(1 - p _ {p a g e}) L _ {\mathrm {m a x}} ^ {2}}{B}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + \frac {2 b ^ {2}}{p _ {p a g e}} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2}, \quad \forall i \in [ n ]. \\ \end{array}
$$
Proof. First, we prove the first inequality of the theorem:
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \mathrm {E} _ {p _ {\text {p a g e}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right]\right] \\ = p _ {\text {p a g e}} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {1} ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] \right] + (1 - p _ {\text {p a g e}}) \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {2} ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] \right]. \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \mathrm {E} _ {\mathrm {p} _ {\mathrm {a}}} \left[ h _ {i, 1} ^ {t + 1} \right] \right] = \\ = p _ {\mathrm {a}} h _ {i} ^ {t} + \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - \frac {b}{p _ {\text {p a g e}}} \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) + \left(1 - p _ {\mathrm {a}}\right) h _ {i} ^ {t} \\ = h _ {i} ^ {t} + \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - \frac {b}{p _ {\text {p a g e}}} \left(h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t})\right). \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \mathrm {E} _ {\mathrm {p} _ {\mathrm {a}}} \left[ h _ {i, 2} ^ {t + 1} \right] \right] = \\ = p _ {\mathrm {a}} h _ {i} ^ {t} + \mathrm {E} _ {B} \left[ \frac {1}{B} \sum_ {j \in I _ {i} ^ {t}} \left(\nabla f _ {i j} \left(x ^ {t + 1}\right) - \nabla f _ {i j} \left(x ^ {t}\right)\right) \right] + (1 - p _ {\mathrm {a}}) h _ {i} ^ {t} \\ = h _ {i} ^ {t} + \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}), \\ \end{array}
$$
we obtain
$$
\begin{array}{l} \left. \right. \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \mathrm {E} _ {p _ {\text {p a g e}}} \left[\left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right]\right]\right] \\ \stackrel {(1 6)} {=} p _ {\text {p a g e}} \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {1} ^ {t + 1} - \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {1} ^ {t + 1} \right] \right\| ^ {2} \right] + (1 - p _ {\text {p a g e}}) \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {2} ^ {t + 1} - \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {2} ^ {t + 1} \right] \right] \right\| ^ {2} \right] \right] \\ \end{array}
$$
$$
\begin{array}{l} + p _ {\text {p a g e}} \left\| \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {1} ^ {t + 1} \right] - \nabla f (x ^ {t + 1}) \right\| ^ {2} + \left(1 - p _ {\text {p a g e}}\right) \left\| \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {2} ^ {t + 1} \right] \right] - \nabla f (x ^ {t + 1}) \right\| ^ {2} \\ = p _ {\text {p a g e}} \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {1} ^ {t + 1} - \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {1} ^ {t + 1} \right] \right\| ^ {2} \right] + (1 - p _ {\text {p a g e}}) \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {2} ^ {t + 1} - \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {2} ^ {t + 1} \right] \right] \right\| ^ {2} \right] \right. \\ + \left(p _ {\text {p a g e}} \left(1 - \frac {b}{p _ {\text {p a g e}}}\right) ^ {2} + \left(1 - p _ {\text {p a g e}}\right)\right) \left\| h ^ {t} - \nabla f \left(x ^ {t}\right) \right\| ^ {2}. \tag {24} \\ \end{array}
$$
Next, we consider  $\mathrm{E}_{p_{\mathrm{a}}}\left[\left\| h_1^{t + 1} - \mathrm{E}_{p_{\mathrm{a}}}\left[h_1^{t + 1}\right]\right\| ^2\right]$ . We can use Lemma 1 with  $r_i = h_i^t$  and  $s_i = k_{i,1}^{t + 1}$  to obtain
$$
\begin{array}{l} \left. \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {1} ^ {t + 1} - \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {1} ^ {t + 1} \right] \right\| ^ {2} \right] \right. \\ \leq \frac {1}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \left\| k _ {i, 1} ^ {t + 1} - k _ {i, 1} ^ {t + 1} \right\| ^ {2} + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| k _ {i, 1} ^ {t + 1} \right\| ^ {2} \\ = \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - \frac {b}{p _ {\text {p a g e}}} \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ \stackrel {(1 5)} {\leq} \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}. \\ \end{array}
$$
From Assumption 3, we have
$$
\begin{array}{l} \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {1} ^ {t + 1} - \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {1} ^ {t + 1} \right] \right\| ^ {2} \right] \\ \leq \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}. \tag {25} \\ \end{array}
$$