SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \stackrel {(1 4)} {=} - \gamma_ {1} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}} \left[ \right.\left\langle \right. F \left(\boldsymbol {x} ^ {k} - \gamma_ {1} \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right)\right), \boldsymbol {x} ^ {k} - \gamma_ {1} \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) - \boldsymbol {x} ^ {*}\left. \right)\left. \right]\left. \right. \\ - \frac {\gamma_ {1} ^ {2}}{2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left[ \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \right] - \frac {\gamma_ {1} ^ {2}}{2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}} \left[ \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) \right\| ^ {2} \right] \\ + \frac {\gamma_ {1} ^ {2}}{2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left[ \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) \right\| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {\text {(Q S M), (1 6)}} {\leq} - \mu \gamma_ {1} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left[ \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} - \gamma_ {1} \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2} \right] - \frac {\gamma_ {1} ^ {2}}{2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}} \left[ \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2} \right] \\ + \frac {4 \gamma_ {1} ^ {2}}{2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}} \left[ \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2} \right] \\ + \frac {4 \gamma_ {1} ^ {2}}{2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}} \left[ \left\| F (\boldsymbol {x} ^ {k}) - F \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \right] \\ + \frac {4 \gamma_ {1} ^ {2}}{2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}} \left[ \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) - F (\boldsymbol {x} ^ {k}) \right\| ^ {2} \right] \\ + \frac {4 \gamma_ {1} ^ {2}}{2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left[ \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - F \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 7), (\text {L i p}), L e m.} {\leq} \stackrel {D. 1, C o r. 2} {-} - \frac {\mu \gamma_ {1}}{2} \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - \frac {\gamma_ {1} ^ {2}}{2} (1 - 2 \gamma_ {1} \mu - 4 \gamma_ {1} ^ {2} L ^ {2}) \mathbb {E} _ {\boldsymbol {\xi} ^ {k}} \left[ \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) \right\| ^ {2} \right] \\ + \frac {4 \gamma_ {1} ^ {2} \sigma^ {2}}{2 G} + \frac {4 \gamma_ {1} ^ {2} \sigma^ {2}}{2 G} + 4 \gamma_ {1} ^ {2} c \delta \rho^ {2} \\ \stackrel {(3 4)} {\leq} \quad - \frac {\mu \gamma_ {1}}{2} \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - \frac {\gamma_ {1} ^ {2}}{2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}} \left[ \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2} \right] + \frac {4 \gamma_ {1} ^ {2} \sigma^ {2}}{G} + 4 \gamma_ {1} ^ {2} c \delta \rho^ {2} \\ \end{array}
$$
So one have
$$
- \widehat {P} _ {k} \leq - \frac {\mu \gamma_ {1}}{2} \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - \frac {\gamma_ {1} ^ {2}}{4} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}} \left[ \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2} \right] + \frac {4 \gamma_ {1} ^ {2} \sigma^ {2}}{G} + \frac {9 \gamma_ {1} ^ {2} c \delta \rho^ {2}}{2}
$$
$$
- \widehat {P} _ {k} \leq - \frac {\mu \gamma_ {1}}{2} \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {4 \gamma_ {1} ^ {2} \sigma^ {2}}{G} + 4 \gamma_ {1} ^ {2} c \delta \rho^ {2}
$$
D.2.2 Quasi-Strongly Monotone Case
Combining Lemmas D.2 and D.3, we get the following result.
Theorem (Theorem 2 duplicate). Let Assumptions 1, 2, 3 and 4 hold. Then after  $T$  iterations SEG-RA (Algorithm 2) with  $(\delta, c)$ -RAGG,  $\gamma_1 \leq \frac{1}{2\mu + 2L}$  and  $\beta = \gamma_2 / \gamma_1 \leq 1/4$  outputs  $\mathbf{x}^T$  such that
$$
\mathbb {E} \left\| \boldsymbol {x} ^ {T} - \boldsymbol {x} ^ {*} \right\| ^ {2} \leq \left(1 - \frac {\mu \beta \gamma_ {1}}{4}\right) ^ {T} \left\| \boldsymbol {x} ^ {0} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {8 \gamma_ {1} \sigma^ {2}}{\mu \beta G} + 8 c \delta \rho^ {2} \left(\frac {\gamma_ {1}}{\beta \mu} + \frac {2}{\mu^ {2}}\right),
$$
Proof of Theorem 2. Since  $\pmb{x}^{k + 1} = \pmb{x}^k -\gamma_2\widehat{\pmb{g}}_{\pmb{\eta}^k}\left(\widetilde{\pmb{x}}^k\right)$ , we have
$$
\begin{array}{l} \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} = \left\| \boldsymbol {x} ^ {k} - \gamma_ {2} \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \boldsymbol {x} ^ {*} \right\| ^ {2} \\ = \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \gamma_ {2} \left\langle \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\rangle + \gamma_ {2} ^ {2} \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \\ \leq \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \gamma_ {2} \left\langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\rangle + 2 \gamma_ {2} ^ {2} \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \\ + \quad 2 \gamma_ {2} ^ {2} \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} + 2 \gamma_ {2} \left\langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\rangle \\ \leq (1 + \lambda) \| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \| ^ {2} - 2 \gamma_ {2} \left\langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\rangle + 2 \gamma_ {2} ^ {2} \| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) \| ^ {2} \\ + \gamma_ {2} ^ {2} \left(2 + \frac {1}{\lambda}\right) \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \\ \end{array}
$$
Taking the expectation, conditioned on  $\pmb{x}^k$ ,
$$
\begin{array}{l} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \leq (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \beta \gamma_ {1} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \rangle \\ + 2 \beta^ {2} \gamma_ {1} ^ {2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} + \gamma_ {2} ^ {2} c \delta \rho^ {2} \left(2 + \frac {1}{\lambda}\right), \\ \end{array}
$$
using the definition of  $\widehat{P}_k = \gamma_1\mathbb{E}_{\pmb{\xi}^k,\pmb{\eta}^k}\left[\left\langle \overline{\pmb{g}}_{\pmb{\eta}^k}(\widetilde{\pmb{x}}^k),\pmb{x}^k -\pmb{x}^*\right\rangle \right]$  we continue our derivation:
$$
\mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left[ \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \right] \tag {36}
$$
$$
\begin{array}{l} = \quad (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \beta \widehat {P} _ {k} + 2 \beta^ {2} \gamma_ {1} ^ {2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \\ + \gamma_ {2} ^ {2} c \delta \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(3 0)} {\leq} \quad (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \beta \widehat {P} _ {k} + 2 \beta^ {2} \left(2 \widehat {P} _ {k} + \frac {8 \gamma_ {1} ^ {2} \sigma^ {2}}{G} + 4 \gamma_ {1} ^ {2} c \delta \rho^ {2}\right) \\ + \gamma_ {2} ^ {2} c \delta \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {0 \leq \beta \leq 1 / 2} {\leq} (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \widehat {P} _ {k} (\beta - 2 \beta^ {2}) + \frac {1 6 \gamma_ {2} ^ {2} \sigma^ {2}}{G} + 8 \gamma_ {2} ^ {2} c \delta \rho^ {2} \\ + \gamma_ {2} ^ {2} c \delta \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(3 5)} {\leq} \quad (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} \\ + 2 \beta (1 - 2 \beta) \left(- \frac {\mu \gamma_ {1}}{2} \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {4 \gamma_ {1} ^ {2} \sigma^ {2}}{G} + 4 \gamma_ {1} ^ {2} c \delta \rho^ {2}\right) \\ + \frac {1 6 \gamma_ {2} ^ {2} \sigma^ {2}}{G} + 8 \gamma_ {2} ^ {2} c \delta \rho^ {2} + \gamma_ {2} ^ {2} c \delta \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \\ \end{array}
$$
$$
\begin{array}{l} \leq \left(1 + \lambda - 2 \beta (1 - 2 \beta) \frac {\mu \gamma_ {1}}{2}\right) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} \\ + \frac {\gamma_ {1} ^ {2} \sigma^ {2}}{G} + \gamma_ {1} ^ {2} c \delta \rho^ {2} + \frac {1 6 \gamma_ {2} ^ {2} \sigma^ {2}}{G} + 8 \gamma_ {2} ^ {2} c \delta \rho^ {2} + \gamma_ {2} ^ {2} c \delta \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \tag {37} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {0 \leq \beta \leq 1 / 4} {\leq} \left(1 + \lambda - \frac {\mu \gamma_ {2}}{2}\right) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {\sigma^ {2}}{G} \left(\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2}\right) \\ + c \delta \rho^ {2} \left(\gamma_ {1} ^ {2} + 1 0 \gamma_ {2} ^ {2} + \frac {\gamma_ {2} ^ {2}}{\lambda}\right) \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {\lambda = \mu \gamma_ {2} / 4} {\leq} \left(1 - \frac {\mu \gamma_ {2}}{4}\right) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {\sigma^ {2}}{G} \left(\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2}\right) \\ + c \delta \rho^ {2} \left(\gamma_ {1} ^ {2} + 1 0 \gamma_ {2} ^ {2} + \frac {4 \gamma_ {2}}{\mu}\right) \\ \end{array}
$$
Next, we take the full expectation from the both sides
$$
\mathbb {E} \left[ \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \right] \leq \left(1 - \frac {\mu \gamma_ {2}}{4}\right) \mathbb {E} \left[ \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} \right] + \frac {\sigma^ {2}}{G} \left(\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2}\right) + c \delta \rho^ {2} \left(\gamma_ {1} ^ {2} + 1 0 \gamma_ {2} ^ {2} + \frac {4 \gamma_ {2}}{\mu}\right). \tag {38}
$$
Unrolling the recurrence, together with the bound on  $\rho$  given by Lemma D.1 we derive the result of the theorem:
$$
\mathbb {E} \left[ \left\| \pmb {x} ^ {K} - \pmb {x} ^ {*} \right\| ^ {2} \right] \leq \left(1 - \frac {\mu \gamma_ {2}}{4}\right) ^ {K} \left\| \pmb {x} ^ {0} - \pmb {x} ^ {*} \right\| ^ {2} + \frac {4 \sigma^ {2} (\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2})}{\mu \gamma_ {2} G} + \frac {4 c \delta \rho^ {2} (\gamma_ {1} ^ {2} + 1 0 \gamma_ {2} ^ {2} + \frac {4 \gamma_ {2}}{\mu})}{\mu \gamma_ {2}}.
$$
Corollary 3. Let assumptions of Theorem 2 hold. Then  $\mathbb{E}\left\| \boldsymbol{x}^T - \boldsymbol{x}^*\right\|^2 \leq \varepsilon$  holds after
$$
T \geq 4 \left(\frac {2}{\beta} + \frac {2 \ell}{\beta \mu} + \frac {1}{3 \beta c \delta G}\right) \ln \frac {3 R ^ {2}}{\varepsilon}
$$
iterations of SEG-RA with  $\gamma_{1} = \min \left(\frac{1}{2\mu + 2L},\frac{1}{2\mu + \frac{\mu}{12c\delta G}}\right)$  and  $b\geq \frac{288c\delta\sigma^2}{\beta\mu^2\varepsilon}$ .
Proof. Next, we plug  $\gamma_{2} = \beta \gamma_{1} \leq \gamma_{1} / 4$  into the result of Theorem 2 and obtain
$$
\mathbb {E} \left[ \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \right] \leq \left(1 - \frac {\mu \beta \gamma_ {1}}{4}\right) \left\| \boldsymbol {x} ^ {0} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {8 \gamma_ {1} ^ {2} \sigma^ {2}}{\mu \beta G} + \frac {8 \gamma_ {1} c \delta \rho^ {2}}{\beta \mu} + \frac {1 6 c \delta \rho^ {2}}{\beta \mu^ {2}}. \tag {39}
$$
If  $\zeta = 0$ ,  $\rho^2 = 24\sigma^2$  the last recurrence can be unrolled as
$$
\mathbb {E} \big \| \pmb {x} ^ {T} - \pmb {x} ^ {*} \big \| ^ {2} \leq \left(1 - \frac {\mu \beta \gamma_ {1}}{4}\right) ^ {T} \big \| \pmb {x} ^ {0} - \pmb {x} ^ {*} \big \| ^ {2} + \frac {8 \gamma_ {1} \sigma^ {2}}{\mu \beta G} + \frac {8 \cdot 2 4 \gamma_ {1} c \delta \sigma^ {2}}{\beta \mu} + \frac {1 6 \cdot 2 4 c \delta \sigma^ {2}}{\beta \mu^ {2}}.
$$
D.3 Proofs for M-SGDA-RA
Algorithm 3 M-SGDA-RA
Input: RAGG,  $\gamma$ ,  $\alpha \in [0,1]$
1: for  $t = 0, \ldots$  do
2: for worker  $i \in [n]$  in parallel
3:  $\pmb{g}_i^t\gets \pmb {g}_i(\pmb {x}^t,\pmb {\xi}_i)$  and  $\pmb {m}_i^t\gets (1 - \alpha)\pmb {m}_i^{t - 1} + \alpha \pmb {g}_i^t$
4: send  $m_i^t$  if  $i \in \mathcal{G}$ , else send * if Byzantine
5:  $\widehat{\pmb{m}}^t = \mathrm{RAGG}(\pmb{m}_1^t,\dots,\pmb{m}_n^t)$  and  $\pmb{x}^{t + 1}\gets \pmb{x}^t -\gamma \widehat{\pmb{m}}^t$  aggregate
// worker momentum
// update params using robust
D.3.1 Quasi-Strongly Monotone Case
Theorem (Theorem 3 duplicate). Let Assumptions 1, 2, 4, and 6 hold. Then after  $T$  iterations M-SGDA-RA (Algorithm 3) with  $(\delta, c)$ -RAGG outputs  $\overline{\boldsymbol{x}}^T$  such that
$$
\mathbb {E} \Big [ \big \| \overline {{\boldsymbol {x}}} ^ {T} - \boldsymbol {x} ^ {*} \big \| ^ {2} \Big ] \leq \frac {2 \big \| \boldsymbol {x} ^ {0} - \boldsymbol {x} ^ {*} \big \| ^ {2}}{\mu \gamma \alpha W _ {T}} + \frac {4 c \delta \rho^ {2}}{\mu^ {2} \alpha^ {2}} + \frac {8 \gamma c \delta \rho^ {2}}{\mu \alpha^ {2}} + \frac {6 \gamma \sigma^ {2}}{\mu \alpha^ {2} G},
$$
where  $\overline{\pmb{x}}^T = \frac{1}{W_T}\sum_{t = 0}^T w_t\widehat{\pmb{x}}^t,\quad \widehat{\pmb{x}}^t = \frac{\alpha}{1 - (1 - \alpha)^{t + 1}}\sum_{j = 0}^t (1 - \alpha)^{t - j}\pmb {x}^j,\quad w_t = \left(1 - \frac{\mu\gamma\alpha}{2}\right)^{-t - 1},$  and  $W_{T} = \sum_{t = 0}^{T}w_{t}$  and  $\rho^2 = 24\sigma^2 +12\zeta^2$  by Lemma D.1.
Proof of Theorem 3. Since  $\widehat{\pmb{m}}^t = \widehat{\pmb{m}}^t -\overline{\pmb{m}}^t +\overline{\pmb{m}}^t$  and  $\overline{\pmb{m}}^t = \alpha \overline{\pmb{g}}^t +(1 - \alpha)\overline{\pmb{m}}^{t - 1}$  one has
$$
\begin{array}{l} \left\| \boldsymbol {x} ^ {t + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} = \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} - \gamma \widehat {\boldsymbol {m}} ^ {t} \right\| ^ {2} = \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \gamma \langle \widehat {\boldsymbol {m}} ^ {t}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \rangle + \gamma^ {2} \left\| \widehat {\boldsymbol {m}} ^ {t} \right\| ^ {2} = \\ = \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \gamma \langle \widehat {\boldsymbol {m}} ^ {t} - \overline {{\boldsymbol {m}}} ^ {t}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \rangle + \gamma^ {2} \left\| \widehat {\boldsymbol {m}} ^ {t} \right\| ^ {2} \\ - 2 \gamma \langle \overline {{\boldsymbol {m}}} ^ {t}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \rangle . \\ \end{array}
$$
Next, unrolling the following recursion
$$
\begin{array}{l} \langle \overline {{\boldsymbol {m}}} ^ {t}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \rangle = \alpha \langle \overline {{\boldsymbol {g}}} ^ {t}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \rangle + (1 - \alpha) \langle \overline {{\boldsymbol {m}}} ^ {t - 1}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \rangle \\ = \alpha \langle \overline {{\boldsymbol {g}}} ^ {t}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \rangle + (1 - \alpha) \langle \overline {{\boldsymbol {m}}} ^ {t - 1}, \boldsymbol {x} ^ {t - 1} - \boldsymbol {x} ^ {*} \rangle + (1 - \alpha) \langle \overline {{\boldsymbol {m}}} ^ {t - 1}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {t - 1} \rangle \\ = \alpha \langle \overline {{\boldsymbol {g}}} ^ {t}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \rangle + (1 - \alpha) \langle \overline {{\boldsymbol {m}}} ^ {t - 1}, \boldsymbol {x} ^ {t - 1} - \boldsymbol {x} ^ {*} \rangle + (1 - \alpha) \gamma \langle \overline {{\boldsymbol {m}}} ^ {t - 1}, \widehat {\boldsymbol {m}} ^ {t} \rangle \\ \end{array}
$$
one obtains
$$
\langle \overline {{\boldsymbol {m}}} ^ {t}, \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \rangle = \alpha \sum_ {j = 0} ^ {t} (1 - \alpha) ^ {t - j} \langle \overline {{\boldsymbol {g}}} ^ {j}, \boldsymbol {x} ^ {j} - \boldsymbol {x} ^ {*} \rangle - (1 - \alpha) \gamma \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \langle \overline {{\boldsymbol {m}}} ^ {j - 1}, \widehat {\boldsymbol {m}} ^ {j} \rangle
$$
Applying the latter and (11) for  $\langle \widehat{\pmb{m}}^t -\overline{\pmb{m}}^t,\pmb{x}^t -\pmb{x}^*\rangle$  with  $\lambda = \frac{\mu\gamma\alpha}{2}$  we obtain
$$
2 \alpha \gamma \sum_ {j = 0} ^ {t} (1 - \alpha) ^ {t - j} \left\langle \overline {{\boldsymbol {g}}} ^ {j}, \boldsymbol {x} ^ {j} - \boldsymbol {x} ^ {*} \right\rangle
$$
$$
\begin{array}{l} \leq \left(1 + \frac {\mu \gamma \alpha}{2}\right) \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \right\| ^ {2} - \left\| \boldsymbol {x} ^ {t + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {2 \gamma}{\mu \alpha} \left\| \widehat {\boldsymbol {m}} ^ {t} - \overline {{\boldsymbol {m}}} ^ {t} \right\| ^ {2} \\ + \gamma^ {2} \left\| \widehat {\boldsymbol {m}} ^ {t} \right\| ^ {2} + 2 \gamma^ {2} (1 - \alpha) \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \langle \overline {{\boldsymbol {m}}} ^ {j - 1}, \widehat {\boldsymbol {m}} ^ {j} \rangle \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 1)} {\leq} \left(1 + \frac {\mu \gamma \alpha}{2}\right) \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \right\| ^ {2} - \left\| \boldsymbol {x} ^ {t + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {2 \gamma}{\mu \alpha} \left\| \widehat {\boldsymbol {m}} ^ {t} - \overline {{\boldsymbol {m}}} ^ {t} \right\| ^ {2} \\ + \gamma^ {2} \left\| \widehat {\boldsymbol {m}} ^ {t} \right\| ^ {2} + \gamma^ {2} \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \left\| \widehat {\boldsymbol {m}} ^ {j} \right\| ^ {2} \\ + \gamma^ {2} (1 - \alpha) ^ {2} \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \left\| \overline {{\boldsymbol {m}}} ^ {j - 1} \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(1 2)} {\leq} \left(1 + \frac {\mu \gamma \alpha}{2}\right) \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \right\| ^ {2} - \left\| \boldsymbol {x} ^ {t + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {2 \gamma}{\mu \alpha} \left\| \widehat {\boldsymbol {m}} ^ {t} - \overline {{\boldsymbol {m}}} ^ {t} \right\| ^ {2} \\ + 4 \gamma^ {2} \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \left\| \widehat {\boldsymbol {m}} ^ {j} - \overline {{\boldsymbol {m}}} ^ {j} \right\| ^ {2} \\ + 3 \gamma^ {2} \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \left\| \overline {{\boldsymbol {m}}} ^ {j - 1} \right\| ^ {2} \\ \end{array}
$$
Since  $\overline{\pmb{m}}^t = \alpha \sum_{j=0}^{t} (1 - \alpha)^{t-j} \overline{\pmb{g}}^j$  and hence  $\left\| \overline{\pmb{m}}^t \right\|^2 \leq \alpha \sum_{j=0}^{t} (1 - \alpha)^{t-j} \left\| \overline{\pmb{g}}^j \right\|^2$  one has
$$
\begin{array}{l} 2 \alpha \gamma \sum_ {j = 0} ^ {t} (1 - \alpha) ^ {t - j} \left\langle \bar {\boldsymbol {g}} ^ {j}, \boldsymbol {x} ^ {j} - \boldsymbol {x} ^ {*} \right\rangle \\ \leq \left(1 + \frac {\mu \gamma \alpha}{2}\right) \| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \| ^ {2} - \| \boldsymbol {x} ^ {t + 1} - \boldsymbol {x} ^ {*} \| ^ {2} + \frac {2 \gamma}{\mu \alpha} \| \widehat {\boldsymbol {m}} ^ {t} - \overline {{\boldsymbol {m}}} ^ {t} \| ^ {2} \\ + 4 \gamma^ {2} \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \left\| \widehat {\boldsymbol {m}} ^ {j} - \overline {{\boldsymbol {m}}} ^ {j} \right\| ^ {2} \\ + 3 \gamma^ {2} \alpha \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \sum_ {i = 0} ^ {j} (1 - \alpha) ^ {j - i} \left\| \overline {{\boldsymbol {g}}} ^ {i} \right\| ^ {2}. \\ \end{array}
$$
$$
\begin{array}{l} 2 \alpha \gamma \sum_ {j = 0} ^ {t} (1 - \alpha) ^ {t - j} \left\langle F ^ {j}, \boldsymbol {x} ^ {j} - \boldsymbol {x} ^ {*} \right\rangle \\ \leq \left(1 + \frac {\mu \gamma \alpha}{2}\right) \left\| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \right\| ^ {2} - \mathbb {E} _ {\boldsymbol {\xi}} \left\| \boldsymbol {x} ^ {t + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \\ + \frac {2 \gamma}{\mu \alpha} \mathbb {E} _ {\boldsymbol {\xi}} \left\| \widehat {\boldsymbol {m}} ^ {t} - \overline {{\boldsymbol {m}}} ^ {t} \right\| ^ {2} + 4 \gamma^ {2} \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \mathbb {E} _ {\boldsymbol {\xi}} \left\| \widehat {\boldsymbol {m}} ^ {j} - \overline {{\boldsymbol {m}}} ^ {j} \right\| ^ {2} \\ + 3 \gamma^ {2} \alpha \sum_ {j = 1} ^ {t} (1 - \alpha) ^ {t - j} \sum_ {i = 0} ^ {j} (1 - \alpha) ^ {j - i} \mathbb {E} _ {\boldsymbol {\xi}} \left\| \overline {{\boldsymbol {g}}} ^ {i} \right\| ^ {2}. \\ \end{array}
$$
Next we use Lemma C.1 to bound  $\mathbb{E}_{\boldsymbol{\xi}}\left\| \overline{\boldsymbol{g}}^j\right\|^2$  and Assumption (QSM) to obtain the following bound
$$
- \alpha \sum_ {j = 0} ^ {t} (1 - \alpha) ^ {t - j} \left\langle F ^ {j}, \boldsymbol {x} ^ {j} - \boldsymbol {x} ^ {*} \right\rangle \leq - \alpha \sum_ {j = 0} ^ {t} (1 - \alpha) ^ {t - j} \left\| \boldsymbol {x} ^ {j} - \boldsymbol {x} ^ {*} \right\| ^ {2} \leq - \alpha \mu \| \boldsymbol {x} ^ {t} - \boldsymbol {x} ^ {*} \| ^ {2}.
$$
Gathering the above results we have