SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \nu \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ + \delta \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left| \left| h _ {i j} ^ {t + 1} - \nabla f _ {i j} \left(x ^ {t + 1}\right) \right| \right| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \right. \\ - \nu \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) - \rho \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) - \delta \frac {2 m L _ {\operatorname* {m a x}} ^ {2}}{p _ {\mathrm {a}} B}\left. \right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \left(\gamma + \nu (1 - b) ^ {2}\right) \operatorname {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ \left. + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \nu \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) b ^ {2}}{n p _ {\mathrm {a}} ^ {2}} + \rho (1 - b)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right] \right. \\ \left. \right. + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {2 \nu b ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 \rho b ^ {2}}{p _ {\mathrm {a}} B} + \delta (1 - b)\right) \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
Thus, if we take  $\nu = \frac{\gamma}{b}$ , then  $\gamma + \nu (1 - b)^2 \leq \nu$  and
$$
\begin{array}{l} \operatorname {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \operatorname {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \rho \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ + \delta \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t + 1} - \nabla f _ {i j} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \right. \\ \end{array}
$$
$$
\begin{array}{l} \left. - \left(\frac {2 \gamma L _ {\max} ^ {2}}{b n p _ {\mathrm {a}} B} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}}\right) - \rho \left(\frac {2 L _ {\max} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) - \delta \frac {2 m L _ {\max} ^ {2}}{p _ {\mathrm {a}} B}\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b}{n p _ {\mathrm {a}} ^ {2}} + \rho (1 - b)\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} \right] \\ \left. \right. + \left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {2 \gamma b}{n p _ {\mathrm {a}} B} + \frac {2 \rho b ^ {2}}{p _ {\mathrm {a}} B} + \delta (1 - b)\right) \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
Next, if we take  $\rho = \frac{8b\gamma\omega(2\omega + 1)}{np_{\mathrm{a}}^2} +\frac{2\gamma(p_{\mathrm{a}} - p_{\mathrm{aa}})}{np_{\mathrm{a}}^2}$ , then
$$
\left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b}{n p _ {\mathrm {a}} ^ {2}} + \rho (1 - b)\right) = \rho ,
$$
Due to  $b \leq p_{\mathrm{a}}$  and  $\frac{p_{\mathrm{a}} - p_{\mathrm{aa}}}{p_{\mathrm{a}}} \leq 1$ , we have
$$
\begin{array}{l} \frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {2 \gamma b}{n p _ {\mathrm {a}} B} + \frac {1 6 b ^ {3} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {3} B} + \frac {4 b ^ {2} \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n B p _ {\mathrm {a}} ^ {3}} \\ \leq \frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {2 \gamma b}{n p _ {\mathrm {a}} B} + \frac {1 6 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {4 \gamma b}{n p _ {\mathrm {a}} B} \\ \end{array}
$$
$$
= \frac {2 4 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {6 \gamma b}{n p _ {\mathrm {a}} B}.
$$
Let us take  $\delta = \frac{24b\gamma\omega(2\omega + 1)}{np_{\mathrm{a}}^2B} +\frac{6\gamma}{np_{\mathrm{a}}B}$ . Thus
$$
\left(\frac {8 b ^ {2} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {2 \gamma b}{n p _ {\mathrm {a}} B} + \frac {1 6 b ^ {3} \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {3} B} + \frac {4 b ^ {2} \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n B p _ {\mathrm {a}} ^ {3}} + \delta (1 - b)\right) \leq \delta
$$
Let us simplify the term near  $\mathrm{E}\left[\left\| x^{t + 1} - x^t\right\|^2\right]$ . Due to  $b \leq p_{\mathrm{a}}$ ,  $\frac{p_{\mathrm{a}} - p_{\mathrm{aa}}}{p_{\mathrm{a}}} \leq 1$ , and  $1 - p_{\mathrm{a}} \leq 1$ , we have
$$
\begin{array}{l} \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \\ + \left(\frac {2 \gamma L _ {\max} ^ {2}}{b n p _ {\mathrm {a}} B} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}}\right) \\ + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \left(\frac {2 L _ {\max} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) \\ + \left(\frac {2 4 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {6 \gamma}{n p _ {\mathrm {a}} B}\right) \frac {2 m L _ {\max} ^ {2}}{p _ {\mathrm {a}} B} \\ \leq \frac {1 2 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \\ \end{array}
$$
$$
\begin{array}{l} + \left(\frac {6 \gamma L _ {\max} ^ {2}}{b n p _ {\mathrm {a}} B} + \frac {6 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}}\right) \\ + \left(\frac {2 4 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {6 \gamma}{n p _ {\mathrm {a}} B}\right) \frac {2 m L _ {\max} ^ {2}}{p _ {\mathrm {a}} B} \\ \end{array}
$$
Considering that  $b \leq \frac{p_{\mathrm{a}}B}{m}$  and  $b \geq \frac{p_{\mathrm{a}}B}{2m}$ , we obtain
$$
\begin{array}{l} \frac {4 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) \\ + \left(\frac {2 \gamma L _ {\max} ^ {2}}{b n p _ {\mathrm {a}} B} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}}\right) \\ + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}}\right) \left(\frac {2 L _ {\max} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) \widehat {L} ^ {2}}{p _ {\mathrm {a}}}\right) \\ + \left(\frac {2 4 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {6 \gamma}{n p _ {\mathrm {a}} B}\right) \frac {2 m L _ {\max} ^ {2}}{p _ {\mathrm {a}} B} \\ \leq \frac {3 6 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) + \left(\frac {1 8 \gamma L _ {\operatorname* {m a x}} ^ {2}}{b n p _ {\mathrm {a}} B} + \frac {6 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{b n p _ {\mathrm {a}} ^ {2}}\right) \\ \leq \frac {3 6 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) + \left(\frac {3 6 m \gamma L _ {\operatorname* {m a x}} ^ {2}}{n p _ {\mathrm {a}} ^ {2} B ^ {2}} + \frac {1 2 m \gamma (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) \widehat {L} ^ {2}}{B n p _ {\mathrm {a}} ^ {3}}\right). \\ \end{array}
$$
All in all, we have
$$
\begin{array}{l} \mathrm {E} \left[ f (x ^ {t + 1}) \right] + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t + 1} - h ^ {t + 1} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t + 1} - h _ {i} ^ {t + 1} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ + \left(\frac {2 4 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {6 \gamma}{n p _ {\mathrm {a}} B}\right) \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t + 1} - \nabla f _ {i j} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \| g ^ {t} - h ^ {t} \| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \| g _ {i} ^ {t} - h _ {i} ^ {t} \| ^ {2} \right] \\ \left. \right. - \left(\frac {1}{2 \gamma} - \frac {L}{2} - \frac {3 6 \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} \left(\frac {2 L _ {\operatorname* {m a x}} ^ {2}}{B} + 2 \widehat {L} ^ {2}\right) - \left(\frac {3 6 m \gamma L _ {\operatorname* {m a x}} ^ {2}}{n p _ {\mathrm {a}} ^ {2} B ^ {2}} + \frac {1 2 m \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{B n p _ {\mathrm {a}} ^ {3}}\right)\right) \mathrm {E} \left[ \| x ^ {t + 1} - x ^ {t} \| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left| \left| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right| \right| ^ {2} \right] \\ + \left(\frac {2 4 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {6 \gamma}{n p _ {\mathrm {a}} B}\right) \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} \right]. \\ \end{array}
$$
$$
\begin{array}{l} + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t + 1} - \nabla f _ {i} (x ^ {t + 1}) \right\| ^ {2} \right] \\ + \left(\frac {2 4 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {6 \gamma}{n p _ {\mathrm {a}} B}\right) \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t + 1} - \nabla f _ {i j} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \\ \leq \operatorname {E} [ f (x ^ {t}) ] - \frac {\gamma}{2} \operatorname {E} \left[ \| \nabla f (x ^ {t}) \| ^ {2} \right] \\ + \frac {\gamma (2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\gamma ((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}})}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {\gamma}{b} \mathrm {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\frac {8 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \gamma \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}}\right) \operatorname {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \right] \\ + \left(\frac {2 4 b \gamma \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {6 \gamma}{n p _ {\mathrm {a}} B}\right) \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} (x ^ {t}) \right\| ^ {2} \right]. \\ \end{array}
$$
$$
\begin{array}{l} \Psi^ {t} = \frac {(2 \omega + 1)}{p _ {\mathrm {a}}} \mathrm {E} \left[ \left\| g ^ {t} - h ^ {t} \right\| ^ {2} \right] + \frac {\left((2 \omega + 1) p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}} \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| g _ {i} ^ {t} - h _ {i} ^ {t} \right\| ^ {2} \right] \\ + \frac {1}{b} \operatorname {E} \left[ \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \right] \\ + \left(\frac {8 b \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2}} + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right)}{n p _ {\mathrm {a}} ^ {2}}\right) \mathrm {E} \left[ \frac {1}{n} \sum_ {i = 1} ^ {n} \left| \left| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right| \right| ^ {2} \right] \\ + \left(\frac {2 4 b \omega (2 \omega + 1)}{n p _ {\mathrm {a}} ^ {2} B} + \frac {6}{n p _ {\mathrm {a}} B}\right) \mathrm {E} \left[ \frac {1}{n m} \sum_ {i = 1} ^ {n} \sum_ {j = 1} ^ {m} \left\| h _ {i j} ^ {t} - \nabla f _ {i j} \left(x ^ {t}\right) \right\| ^ {2} \right] \\ \end{array}
$$
E.6 Proof for DASHA-PP-MVR
Let us denote  $\nabla f_{i}(x^{t + 1};\xi_{i}^{t + 1})\coloneqq \frac{1}{B}\sum_{j = 1}^{B}\nabla f_{i}(x^{t + 1};\xi_{ij}^{t + 1}).$
Lemma 10. Suppose that Assumptions 3, 5, 6 and 8 hold. For  $h_i^{t+1}$  and  $k_i^{t+1}$  from Algorithm 1 (DASHA-PP-MVR) we have
$$
\begin{array}{l} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left| \left| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right| \right| ^ {2} \right] \right] \\ \leq \frac {2 b ^ {2} \sigma^ {2}}{n p _ {\mathrm {a}} B} + \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{n p _ {\mathrm {a}} B} + \frac {2 \left(p _ {\mathrm {a}} - p _ {\mathrm {a a}}\right) \widehat {L} ^ {2}}{n p _ {\mathrm {a}} ^ {2}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \\ + \frac {2 (p _ {\mathrm {a}} - p _ {\mathrm {a a}}) b ^ {2}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2} + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left| \left| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right) \right| \right| ^ {2} \right] \right] \\ \leq \frac {2 b ^ {2} \sigma^ {2}}{p _ {\mathrm {a}} B} + \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{p _ {\mathrm {a}} B} + \frac {2 (1 - p _ {\mathrm {a}}) L _ {i} ^ {2}}{p _ {\mathrm {a}}}\right) \| x ^ {t + 1} - x ^ {t} \| ^ {2} \\ + \left(\frac {2 (1 - p _ {\mathrm {a}}) b ^ {2}}{p _ {\mathrm {a}}} + (1 - b) ^ {2}\right) \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}, \quad \forall i \in [ n ]. \\ \end{array}
$$
$$
\operatorname {E} _ {k} \left[ \left\| k _ {i} ^ {t + 1} \right\| ^ {2} \right] \leq \frac {2 b ^ {2} \sigma^ {2}}{B} + \left(\frac {2 (1 - b) ^ {2} L _ {\sigma} ^ {2}}{B} + 2 L _ {i} ^ {2}\right) \left\| x ^ {t + 1} - x ^ {t} \right\| ^ {2} + 2 b ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}) \right\| ^ {2}, \quad \forall i \in [ n ].
$$
Proof. First, let us proof the bound for  $\operatorname{E}_k\left[\operatorname{E}_{p_a}\left[\left\| h^{t + 1} - \nabla f(x^{t + 1})\right\| ^2\right]\right]$ :
$$
\begin{array}{l} \left. \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] \right] \right. \\ = \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h ^ {t + 1} - \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} [ h ^ {t + 1} ] \right] \right\| ^ {2} \right] \right] + \left\| \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} [ h ^ {t + 1} ] \right] - \nabla f (x ^ {t + 1}) \right\| ^ {2}. \\ \end{array}
$$
$$
\operatorname {E} _ {k} \left[ \operatorname {E} _ {p _ {\mathrm {a}}} \left[ h _ {i} ^ {t + 1} \right] \right] = h _ {i} ^ {t} + \operatorname {E} _ {k} \left[ k _ {i} ^ {t + 1} \right] = h _ {i} ^ {t} + \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - b (h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t}))
$$
$$
\begin{array}{l} \left. \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right\| ^ {2} \right] \right] \right. \\ = \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h ^ {t + 1} - \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} [ h ^ {t + 1} ] \right] \right\| ^ {2} \right] \right] + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$
$$
\begin{array}{l} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left| \left| h ^ {t + 1} - \nabla f (x ^ {t + 1}) \right| \right| ^ {2} \right] \right] \\ \leq \frac {1}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \operatorname {E} _ {k} \left[ \left\| k _ {i} ^ {t + 1} - \operatorname {E} _ {k} \left[ k _ {i} ^ {t + 1} \right] \right\| ^ {2} \right] + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \operatorname {E} _ {k} \left[ k _ {i} ^ {t + 1} \right] \right\| ^ {2} + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \\ = \frac {1}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {k} \left[ \right.\left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right)\right)\right. \\ \left. \left. - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 5)} {\leq} \frac {2}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \operatorname {E} _ {k} \left[ \left\| b \left(\nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right)\right) \right\| ^ {2} \right] \\ + \frac {2}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {k} \left[ \left\| (1 - b) \left(\nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2} \\ = \frac {2 b ^ {2}}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \operatorname {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \right] \\ + \frac {2 (1 - b) ^ {2}}{n ^ {2} p _ {\mathrm {a}}} \sum_ {i = 1} ^ {n} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \right] \\ + \frac {p _ {\mathrm {a}} - p _ {\mathrm {a a}}}{n ^ {2} p _ {\mathrm {a}} ^ {2}} \sum_ {i = 1} ^ {n} \left\| \nabla f _ {i} (x ^ {t + 1}) - \nabla f _ {i} (x ^ {t}) - b (h _ {i} ^ {t} - \nabla f _ {i} (x ^ {t})) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h ^ {t} - \nabla f (x ^ {t}) \right\| ^ {2}. \\ \end{array}
$$