$$
\begin{array}{l} = \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} - \mathbb {E} \left\langle \mathbf {x} _ {t} - \mathbf {x} _ {*}, 2 \eta_ {t} \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \Delta_ {s} ^ {t} \right\rangle + \mathbb {E} \left\| \eta_ {t} \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \Delta_ {s} ^ {t} \right\| ^ {2} (71) \\ = \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} - \mathbb {E} \left\langle \mathbf {x} _ {t} - \mathbf {x} _ {*}, 2 \eta_ {t} \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \nabla f _ {s} \left(\mathbf {x} _ {t}, \xi_ {t}\right) \right\rangle (72) \\ + \mathbb {E} \left\langle \mathbf {x} _ {t} - \mathbf {x} _ {*}, 2 \eta_ {t} \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \left(\nabla f _ {s} \left(\mathbf {x} _ {t}, \xi_ {t}\right) - \Delta_ {s} ^ {t}\right) \right\rangle + \mathbb {E} \left\| \eta_ {t} \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \Delta_ {s} ^ {t} \right\| ^ {2} (73) \\ = \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} - \left\langle \mathbf {x} _ {t} - \mathbf {x} _ {*}, 2 \eta_ {t} \sum_ {s \in [ S ]} \mathbb {E} \left[ \lambda_ {s} ^ {t} \right] \nabla f _ {s} (\mathbf {x} _ {t}) \right\rangle (74) \\ + \mathbb {E} \left\langle \mathbf {x} _ {t} - \mathbf {x} _ {*}, 2 \eta_ {t} \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \left(\nabla f _ {s} \left(\mathbf {x} _ {t}, \xi_ {t}\right) - \Delta_ {s} ^ {t}\right) \right\rangle + \mathbb {E} \left\| \eta_ {t} \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \Delta_ {s} ^ {t} \right\| ^ {2} (75) \\ \leq \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} - 2 \eta_ {t} \left(\frac {\mu}{2} \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} + \sum_ {s \in [ S ]} \mathbb {E} \left[ \lambda_ {s} ^ {t} \right] \left(f _ {s} \left(\mathbf {x} _ {t}\right) - f _ {s} \left(\mathbf {x} _ {*}\right)\right)\right) + \epsilon \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} (76) \\ + \frac {1}{4 \epsilon} 4 \eta_ {t} ^ {2} \mathbb {E} \left\| \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \left(\nabla f _ {s} \left(\mathbf {x} _ {t}, \xi_ {t}\right) - \Delta_ {s} ^ {t}\right) \right\| ^ {2} + \eta_ {t} ^ {2} \mathbb {E} \left\| \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \Delta_ {s} ^ {t} \right\| ^ {2} (77) \\ \leq \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} - 2 \eta_ {t} \left(\frac {\mu}{2} \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} + \sum_ {s \in [ S ]} \mathbb {E} \left[ \lambda_ {s} ^ {t} \right] \left(f _ {s} \left(\mathbf {x} _ {t}\right) - f _ {s} \left(\mathbf {x} _ {*}\right)\right)\right) + \epsilon \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} (78) \\ + \frac {1}{4 \epsilon} 4 \eta_ {t} ^ {2} S \sum_ {s \in [ S ]} \mathbb {E} \left[ \left(\lambda_ {s} ^ {t}\right) ^ {2} \left\| \left(\nabla f _ {s} (\mathbf {x} _ {t}, \xi_ {t}) - \Delta_ {s} ^ {t}\right) \right\| ^ {2} \right] + \eta_ {t} ^ {2} \mathbb {E} \left\| \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \Delta_ {s} ^ {t} \right\| ^ {2} (79) \\ \leq \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} - 2 \eta_ {t} \left(\frac {\mu}{2} \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} + \sum_ {s \in [ S ]} \mathbb {E} \left[ \lambda_ {s} ^ {t} \right] \left(f _ {s} \left(\mathbf {x} _ {t}\right) - f _ {s} \left(\mathbf {x} _ {*}\right)\right)\right) + \epsilon \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} (80) \\ + \frac {1}{4 \epsilon} 4 \eta_ {t} ^ {2} S \sum_ {s \in [ S ]} \mathbb {E} \left\| \nabla f _ {s} \left(\mathbf {x} _ {t}, \xi_ {t}\right) - \Delta_ {s} ^ {t} \right\| ^ {2} + \eta_ {t} ^ {2} \mathbb {E} \left\| \sum_ {s \in [ S ]} \lambda_ {s} ^ {t} \Delta_ {s} ^ {t} \right\| ^ {2} (81) \\ \leq \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} - 2 \eta_ {t} \left(\frac {\mu}{2} \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} + \sum_ {s \in [ S ]} \mathbb {E} \left[ \lambda_ {s} ^ {t} \right] \left(f _ {s} \left(\mathbf {x} _ {t}\right) - f _ {s} \left(\mathbf {x} _ {*}\right)\right)\right) + \epsilon \left\| \mathbf {x} _ {t} - \mathbf {x} _ {*} \right\| ^ {2} (82) \\ + \frac {1}{4 \epsilon} 4 \eta_ {t} ^ {2} S ^ {2} \left(\alpha \eta_ {L} ^ {2} K ^ {2} D ^ {2} + \beta \sigma^ {2}\right) + \eta_ {t} ^ {2} S ^ {2} D ^ {2} (83) \\ \leq \left(1 - \frac {\eta_ {t} \mu}{2}\right) \| \mathbf {x} _ {t} - \mathbf {x} _ {*} \| ^ {2} - 2 \eta_ {t} \left(\sum_ {s \in [ S ]} \mathbb {E} \left[ \lambda_ {s} ^ {t} \right] \left(f _ {s} \left(\mathbf {x} _ {t}\right) - f _ {s} \left(\mathbf {x} _ {*}\right)\right)\right) (84) \\ + \frac {2}{\mu} \eta_ {t} S ^ {2} \left(\alpha \eta_ {L} ^ {2} K ^ {2} D ^ {2} + \beta \sigma^ {2}\right) + \eta_ {t} ^ {2} S ^ {2} D ^ {2}, (85) \\ \end{array}
$$