$$
\begin{array}{l} \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left| \left| h _ {i} ^ {t + 1} - \nabla f _ {i} \left(x ^ {t + 1}\right) \right| \right| ^ {2} \right] \right] \\ = \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i} ^ {t + 1} - \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {i} ^ {t + 1} \right] \right] \right\| ^ {2} \right] \right] \\ + \left\| \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ h _ {i} ^ {t + 1} \right] \right] - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \\ = \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i} ^ {t + 1} - \left(h _ {i} ^ {t} + \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \right] \\ + \left\| h _ {i} ^ {t} + \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) - \nabla f _ {i} \left(x ^ {t + 1}\right) \right\| ^ {2} \\ = \mathrm {E} _ {k} \left[ \mathrm {E} _ {p _ {\mathrm {a}}} \left[ \left\| h _ {i} ^ {t + 1} - \left(h _ {i} ^ {t} + \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \right. \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ = p _ {\mathrm {a}} \mathrm {E} _ {k} \left[ \left\| h _ {i} ^ {t} + \frac {1}{p _ {\mathrm {a}}} k _ {i} ^ {t + 1} - \left(h _ {i} ^ {t} + \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ \left. + \left(1 - p _ {\mathrm {a}}\right) \left\| h _ {i} ^ {t} - \left(h _ {i} ^ {t} + \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right. \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ = p _ {\mathrm {a}} \mathrm {E} _ {k} \left[ \left\| \frac {1}{p _ {\mathrm {a}}} k _ {i} ^ {t + 1} - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ + \left(1 - p _ {\mathrm {a}}\right) \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ \stackrel {(1 6)} {=} \frac {1}{p _ {\mathrm {a}}} \mathrm {E} _ {k} \left[ \left\| k _ {i} ^ {t + 1} - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ + \frac {\left(1 - p _ {\mathrm {a}}\right) ^ {2}}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + \left(1 - p _ {\mathrm {a}}\right) \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ = \frac {1}{p _ {\mathrm {a}}} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right)\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ + \frac {1 - p _ {\mathrm {a}}}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ = \frac {1}{p _ {\mathrm {a}}} \mathrm {E} _ {k} \left[ \left\| b \left(\nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t + 1}\right)\right) + (1 - b) \left(\nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right)\right) \right\| ^ {2} \right] \\ + \frac {1 - p _ {\mathrm {a}}}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2} \\ \stackrel {(1 5)} {\leq} \frac {2 b ^ {2}}{p _ {\mathrm {a}}} \mathrm {E} _ {k} \left[ \| \nabla f _ {i} (x ^ {t + 1}; \xi_ {i} ^ {t + 1}) - \nabla f _ {i} (x ^ {t + 1}) \| ^ {2} \right] \\ + \frac {2 (1 - b) ^ {2}}{p _ {\mathrm {a}}} \mathrm {E} _ {k} \left[ \left\| \nabla f _ {i} \left(x ^ {t + 1}; \xi_ {i} ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}; \xi_ {i} ^ {t + 1}\right) - \left(\nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \right] \\ + \frac {1 - p _ {\mathrm {a}}}{p _ {\mathrm {a}}} \left\| \nabla f _ {i} \left(x ^ {t + 1}\right) - \nabla f _ {i} \left(x ^ {t}\right) - b \left(h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right)\right) \right\| ^ {2} \\ + (1 - b) ^ {2} \left\| h _ {i} ^ {t} - \nabla f _ {i} \left(x ^ {t}\right) \right\| ^ {2}. \\ \end{array}
$$