SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \operatorname {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \operatorname {E} \left[ \left\| g ^ {t + 1} - h ^ {t + 1} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \right\| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} \left[ f (x ^ {t}) - \frac {\gamma}{2} \left\| \nabla f (x ^ {t}) \right\| ^ {2} - \left(\frac {1}{2 \gamma} - \frac {L}{2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + \gamma \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(2 \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right) \right. \\ \end{array}
$$
$$
\begin{array}{l} \left. - \nu \left(\frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{n p _ {\mathrm {a}} B}\right) - \rho \left(\frac {2 \left(1 - p _ {\mathrm {a}}\right) \widehat {L} ^ {2}}{p _ {\mathrm {a}}} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{p _ {\mathrm {a}} B}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ \left. + \left(\gamma + \nu \left(p _ {\text {p a g e}} \left(1 - \frac {b}{p _ {\text {p a g e}}}\right) ^ {2} + (1 - p _ {\text {p a g e}})\right)\right) E \left[ \left\| h ^ {t} - \nabla f \left(x ^ {t}\right) \right\| ^ {2} \right] \right. \\ + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} + \frac {2 \nu \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}}\right) \\ \left. + \rho \left(\frac {2 \left(1 - p _ {\mathrm {a}}\right) b ^ {2}}{p _ {\mathrm {a}} p _ {\text {p a g e}}} + p _ {\text {p a g e}} \left(1 - \frac {b}{p _ {\text {p a g e}}}\right) ^ {2} + (1 - p _ {\text {p a g e}})\right)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \right]. \\ \end{array}
$$
Due to  $b = \frac{p_{\mathrm{page}} p_{\mathrm{a}}}{2 - p_{\mathrm{a}}} \leq p_{\mathrm{page}}$ , one can show that  $\left(p_{\mathrm{page}} \left(1 - \frac{b}{p_{\mathrm{page}}}\right)^2 + (1 - p_{\mathrm{page}})\right) \leq 1 - b$ . Thus, if we take  $\nu = \frac{\gamma}{b}$ , then
$$
\left. \right.\left(\gamma + \nu \left(p _ {\text {p a g e}} \left(1 - \frac {b}{p _ {\text {p a g e}}}\right) ^ {2} + (1 - p _ {\text {p a g e}})\right)\right) \leq \gamma + \nu (1 - b) = \nu ,
$$
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(2 \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right) \right. \\ \left. - \frac {\gamma}{b} \left(\frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{n p _ {\mathrm {a}} B}\right) - \rho \left(\frac {2 \left(1 - p _ {\mathrm {a}}\right) \widehat {L} ^ {2}}{p _ {\mathrm {a}}} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{p _ {\mathrm {a}} B}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}}\right) \\ \left. + \rho \left(\frac {2 \left(1 - p _ {\mathrm {a}}\right) b ^ {2}}{p _ {\mathrm {a}} p _ {\mathrm {p a g e}}} + p _ {\mathrm {p a g e}} \left(1 - \frac {b}{p _ {\mathrm {p a g e}}}\right) ^ {2} + (1 - p _ {\mathrm {p a g e}})\right)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \right]. \\ \end{array}
$$
Next, with the choice of  $b = \frac{p_{\mathrm{page}} p_{\mathrm{a}}}{2 - p_{\mathrm{a}}}$ , we ensure that
$$
\left(\frac {2 \left(1 - p _ {\mathrm {a}}\right) b ^ {2}}{p _ {\mathrm {a}} p _ {\text {p a g e}}} + p _ {\text {p a g e}} \left(1 - \frac {b}{p _ {\text {p a g e}}}\right) ^ {2} + (1 - p _ {\text {p a g e}})\right) \leq 1 - b.
$$
If we take  $\rho = \frac{8b\gamma\omega(2\omega + 1)}{np_{\mathrm{a}}^2p_{\mathrm{page}}} +\frac{2\gamma(p_{\mathrm{a}} - p_{\mathrm{aa}})}{np_{\mathrm{a}}^2p_{\mathrm{page}}}$ , then
$$
\left. \right.\left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} + \rho \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}} p _ {\text {p a g e}}} + p _ {\text {p a g e}} \left(1 - \frac {b}{p _ {\text {p a g e}}}\right) ^ {2} + (1 - p _ {\text {p a g e}})\right)\right) \leq \rho ,
$$
$$
\mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right]
$$
$$
\begin{array}{l} + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(2 \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right) \right. \\ - \frac {\gamma}{b n p _ {\mathrm {a}}} \left(2 \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right) \\ \left. - \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {3} p _ {\text {p a g e}}} + \frac {2 \gamma \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}}\right) \left(2 \left(1 - p _ {\mathrm {a}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {p a g e}}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {p a g e}}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
Let us simplify the inequality. First, due to  $b \geq \frac{p_{\mathrm{page}} p_{\mathrm{a}}}{2}$ , we have
$$
\frac {\gamma}{b n p _ {\mathrm {a}}} \left(2 \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\mathrm {p a g e}}) L _ {\mathrm {m a x}} ^ {2}}{B}\right) \leq \frac {4 \gamma}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {p a g e}}} \left(\left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\mathrm {p a g e}}) L _ {\mathrm {m a x}} ^ {2}}{B}\right).
$$
Second, due to  $b \leq p_{\mathrm{a}}p_{\mathrm{page}}$  and  $p_{\mathrm{aa}} \leq p_{\mathrm{a}}^2$ , we get
$$
\begin{array}{l} \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {3} p _ {\text {p a g e}}} + \frac {2 \gamma \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}}\right) \left(2 \left(1 - p _ {\mathrm {a}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right) \\ \leq \left(\frac {8 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {p a g e}}}\right) \left(2 \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\mathrm {p a g e}}) L _ {\max } ^ {2}}{B}\right) \\ \leq \frac {1 6 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right) \\ + \frac {4 \gamma \left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} \left(\left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right) \\ \leq \frac {1 6 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right) \\ + \frac {4 \gamma}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} \left(\left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max} ^ {2}}{B}\right). \\ \end{array}
$$
Combining all bounds together, we obtain the following simplified inequality:
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \left\| \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t} - h ^ {t} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t} - h _ {i} ^ {t} \| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {2 4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right) \right. \\ \left. - \frac {8 \gamma}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} \left(\left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\text {p a g e}}) L _ {\max } ^ {2}}{B}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {p a g e}}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {p a g e}}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
$$
\begin{array}{l} \operatorname {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \operatorname {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
$$
\begin{array}{l} \Psi^ {t} = \frac {(2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\left((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {1}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] + \left(\frac {8 b \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}} + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2} p _ {\text {p a g e}}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right] \\ \end{array}
$$
$$
T := \mathcal {O} \left(\frac {\Delta_ {0}}{\varepsilon} \left[ L + \frac {\omega}{p _ {\mathrm {a}} \sqrt {n}} \left(\widehat {L} + \frac {L _ {\operatorname* {m a x}}}{\sqrt {B}}\right) + \frac {1}{p _ {\mathrm {a}}} \sqrt {\frac {m}{n}} \left(\frac {\mathbb {1} _ {p _ {\mathrm {a}}} \widehat {L}}{\sqrt {B}} + \frac {L _ {\operatorname* {m a x}}}{B}\right) \right]\right) \tag {10}
$$
Proof. In the view of Theorem 3, it is enough to do
$$
T := \mathcal {O} \left(\frac {\Delta_ {0}}{\varepsilon} \left[ L + \sqrt {\frac {\omega^ {2}}{n p _ {\mathrm {a}} ^ {2}} \left(\widehat {L} ^ {2} + \frac {(1 - p _ {\mathrm {p a g e}}) L _ {\mathrm {m a x}} ^ {2}}{B}\right) + \frac {1}{n p _ {\mathrm {a}} ^ {2} p _ {\mathrm {p a g e}}} \left(\left(1 - \frac {p _ {\mathrm {a a}}}{p _ {\mathrm {a}}}\right) \widehat {L} ^ {2} + \frac {(1 - p _ {\mathrm {p a g e}}) L _ {\mathrm {m a x}} ^ {2}}{B}\right)} \right]\right)
$$
steps to get  $\varepsilon$ -solution. Using the choice of  $p_{\mathrm{mega}}$  and the definition of  $\mathbb{1}_{p_a}$ , we can get (10).
Note that the expected number of gradients calculations at each communication round equals  $p_{\mathrm{mega}} m + (1 - p_{\mathrm{mega}}) B = \frac{2mB}{m + B} \leq 2B$ .
Corollary 2. Suppose that assumptions of Corollary 1 hold,  $B \leq \min \left\{\frac{1}{p_{\mathrm{a}}}\sqrt{\frac{m}{n}},\frac{L_{\max}^{2}}{\mathbb{1}_{p_{\mathrm{a}}}^{2}\widehat{L}^{2}}\right\}^{8}$ , and we use the unbiased compressor RandK with  $K = \Theta \left(\frac{Bd}{\sqrt{m}}\right)$ . Then the communication complexity of
$^{8}$  If  $\mathbb{1}_{p_{\mathrm{a}}} = 0$ , then  $\frac{L_{\sigma}^{2}}{\mathbb{1}_{p_{\mathrm{a}}}^{2}\hat{L}^{2}} = +\infty$
Algorithm 1 is  $\mathcal{O}\left(d + \frac{L_{\max}\Delta_0d}{p_{\mathrm{a}}\varepsilon\sqrt{n}}\right)$ , and the expected number of gradient calculations per node equals  $\mathcal{O}\left(m + \frac{L_{\max}\Delta_0\sqrt{m}}{p_{\mathrm{a}}\varepsilon\sqrt{n}}\right)$ .
Proof. The communication complexity equals
$$
\mathcal {O} (d + K T) = \mathcal {O} \left(d + \frac {\Delta_ {0}}{\varepsilon} \left[ K L + K \frac {\omega}{p _ {\mathrm {a}} \sqrt {n}} \left(\widehat {L} + \frac {L _ {\max}}{\sqrt {B}}\right) + K \frac {1}{p _ {\mathrm {a}}} \sqrt {\frac {m}{n}} \left(\frac {\mathbb {1} _ {p _ {\mathrm {a}}} \widehat {L}}{\sqrt {B}} + \frac {L _ {\max}}{B}\right) \right]\right).
$$
Since  $B \leq \frac{L_{\max}^2}{\mathbb{1}_{p_a}^2 \widehat{L}^2}$ , we have  $\frac{\mathbb{1}_{p_a} \widehat{L}}{\sqrt{B}} + \frac{L_{\max}}{B} \leq \frac{2L_{\max}}{B}$  and
$$
\mathcal {O} (d + K T) = \mathcal {O} \left(d + \frac {\Delta_ {0}}{\varepsilon} \left[ K L + K \frac {\omega}{p _ {\mathrm {a}} \sqrt {n}} \left(\widehat {L} + \frac {L _ {\max}}{\sqrt {B}}\right) + K \frac {1}{p _ {\mathrm {a}}} \sqrt {\frac {m}{n}} \frac {L _ {\max}}{B} \right]\right).
$$
Note that  $K = \Theta\left(\frac{Bd}{\sqrt{m}}\right) = \mathcal{O}\left(\frac{d}{p_{\mathrm{a}}\sqrt{n}}\right)$  and  $\omega + 1 = \frac{d}{K}$  due to Theorem 6, thus
$$
\begin{array}{l} \mathcal {O} (d + K T) = \mathcal {O} \left(d + \frac {\Delta_ {0}}{\varepsilon} \left[ \frac {d}{p _ {\mathrm {a}} \sqrt {n}} L + \frac {d}{p _ {\mathrm {a}} \sqrt {n}} \left(\widehat {L} + \frac {L _ {\max }}{\sqrt {B}}\right) + \frac {d}{p _ {\mathrm {a}} \sqrt {n}} L _ {\max } \right]\right) \\ = \mathcal {O} \left(d + \frac {L _ {\max} \Delta_ {0} d}{p _ {\mathrm {a}} \varepsilon \sqrt {n}}\right). \\ \end{array}
$$
Using the same reasoning, the expected number of gradient calculations per node equals
$$
\begin{array}{l} \mathcal {O} (m + B T) = \mathcal {O} \left(m + \frac {\Delta_ {0}}{\varepsilon} \left[ B L + B \frac {\omega}{p _ {\mathrm {a}} \sqrt {n}} \left(\widehat {L} + \frac {L _ {\max }}{\sqrt {B}}\right) + B \frac {1}{p _ {\mathrm {a}}} \sqrt {\frac {m}{n}} \left(\frac {\mathbb {1} _ {p _ {\mathrm {a}}} \widehat {L}}{\sqrt {B}} + \frac {L _ {\max }}{B}\right) \right]\right) \\ = \mathcal {O} \left(m + \frac {\Delta_ {0}}{\varepsilon} \left[ B L + B \frac {d}{K p _ {\mathrm {a}} \sqrt {n}} \left(\widehat {L} + \frac {L _ {\max }}{\sqrt {B}}\right) + B \frac {1}{p _ {\mathrm {a}}} \sqrt {\frac {m}{n}} \frac {L _ {\max }}{B} \right]\right) \\ = \mathcal {O} \left(m + \frac {\Delta_ {0}}{\varepsilon} \left[ \frac {1}{p _ {\mathrm {a}}} \sqrt {\frac {m}{n}} L + \frac {\sqrt {m}}{p _ {\mathrm {a}} \sqrt {n}} \left(\widehat {L} + \frac {L _ {\max}}{\sqrt {B}}\right) + \frac {1}{p _ {\mathrm {a}}} \sqrt {\frac {m}{n}} L _ {\max} \right]\right) \\ = \mathcal {O} \left(m + \frac {L _ {\operatorname* {m a x}} \Delta_ {0} \sqrt {m}}{p _ {\mathrm {a}} \varepsilon \sqrt {n}}\right). \\ \end{array}
$$
E.5 Proof for DASHA-PP-FINITE-MVR
Lemma 9. Suppose that Assumptions 3, 4, and 8 hold. For  $h_i^{t+1}$ ,  $h_{ij}^{t+1}$  and  $k_i^{t+1}$  from Algorithm 1 (DASHA-PP-FINITE-MVR) we have
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] \right] \\ \leq \left(\frac {2 L _ {\mathrm {m a x}} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + \frac {2 b ^ {2}}{n ^ {2} p _ {\mathrm {a}} B m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} _ {B} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \right] \\ \leq \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) L _ {i} ^ {2}}{p _ {\mathrm {a}}}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} \\ + \frac {2 b ^ {2}}{p _ {\mathrm {a}} B m} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} + \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2}, \quad \forall i \in [ n ]. \\ \end{array}
$$