SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} = \left\| \boldsymbol {x} ^ {k} - \gamma_ {2} \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \boldsymbol {x} ^ {*} \right\| ^ {2} \\ = \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \gamma_ {2} \langle \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \rangle + \gamma_ {2} ^ {2} \| \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) \| ^ {2} \\ \leq \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \gamma_ {2} \left\langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\rangle + 2 \gamma_ {2} ^ {2} \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \\ + 2 \gamma_ {2} ^ {2} \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} + 2 \gamma_ {2} \langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \rangle \\ \leq (1 + \lambda) \| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \| ^ {2} - 2 \gamma_ {2} \left\langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\rangle + 2 \gamma_ {2} ^ {2} \| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \| ^ {2} \\ + \gamma_ {2} ^ {2} \left(2 + \frac {1}{\lambda}\right) \left\| \bar {\mathbf {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) - \hat {\mathbf {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\tilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \leq (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \beta \gamma_ {1} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \rangle \\ + 2 \beta^ {2} \gamma_ {1} ^ {2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} + \gamma_ {2} ^ {2} \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \mathbb {1} _ {k}, \\ \end{array}
$$
using the definition of  $\widehat{P}_k = \gamma_1\mathbb{E}_{\pmb{\xi}^k,\pmb{\eta}^k}\left[\langle \overline{\pmb{g}}_{\pmb{\eta}^k}(\widetilde{\pmb{x}}^k),\pmb{x}^k -\pmb{x}^*\rangle \right]$  , we continue our derivation:
$$
\mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left[ \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \right]
$$
$$
\begin{array}{l} = (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \beta \widehat {P} _ {k} + 2 \beta^ {2} \gamma_ {1} ^ {2} \mathbb {E} _ {\boldsymbol {\xi} ^ {k}, \boldsymbol {\eta} ^ {k}} \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \\ + \gamma_ {2} ^ {2} \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \mathbb {1} _ {k} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(6 9)} {\leq} \quad (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \beta \widehat {P} _ {k} + 2 \beta^ {2} \left(2 \widehat {P} _ {k} + \frac {8 \gamma_ {1} ^ {2} \sigma^ {2}}{G} + 4 \gamma_ {1} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}}\right) \\ + \gamma_ {2} ^ {2} \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \mathbb {1} _ {k} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {0 \leq \beta <   1 / 2} {\leq} (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} - 2 \widehat {P} _ {k} (\beta - 2 \beta^ {2}) + \frac {1 6 \gamma_ {2} ^ {2} \sigma^ {2}}{G} + 8 \gamma_ {2} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}} \\ + \gamma_ {2} ^ {2} \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \mathbb {1} _ {k} \\ \end{array}
$$
$$
\stackrel {(7 4)} {\leq} \quad (1 + \lambda) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2}
$$
$$
+ 2 \beta (1 - 2 \beta) \left(- \frac {\mu \gamma_ {1}}{2} \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {4 \gamma_ {1} ^ {2} \sigma^ {2}}{G} + 4 \gamma_ {1} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}}\right)
$$
$$
+ \frac {1 6 \gamma_ {2} ^ {2} \sigma^ {2}}{G} + 8 \gamma_ {2} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}} + \gamma_ {2} ^ {2} \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \mathbb {1} _ {k}
$$
$$
\begin{array}{l} \leq \left(1 + \lambda - 2 \beta (1 - 2 \beta) \frac {\mu \gamma_ {1}}{2}\right) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} \\ + \frac {\gamma_ {1} ^ {2} \sigma^ {2}}{G} + \gamma_ {1} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}} + \frac {1 6 \gamma_ {2} ^ {2} \sigma^ {2}}{G} + 8 \gamma_ {2} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}} + \gamma_ {2} ^ {2} \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \mathbb {1} _ {k} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {0 \leq \beta \leq 1 / 4} {\leq} \left(1 + \lambda - \frac {\mu \gamma_ {2}}{2}\right) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} \\ + \frac {\sigma^ {2}}{G} \left(\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2}\right) + \gamma_ {1} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}} + 8 \gamma_ {2} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}} + \gamma_ {2} ^ {2} \rho^ {2} \left(2 + \frac {1}{\lambda}\right) \mathbb {1} _ {k} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {\lambda = \mu \gamma_ {2} / 4} {\leq} \left(1 - \frac {\mu \gamma_ {2}}{4}\right) \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} \\ + \frac {\sigma^ {2}}{G} \left(\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2}\right) + \gamma_ {1} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}} + 8 \gamma_ {2} ^ {2} \rho^ {2} \mathbb {1} _ {k - \frac {1}{2}} + \gamma_ {2} ^ {2} \rho^ {2} \left(2 + \frac {4}{\mu \gamma_ {2}}\right) \mathbb {1} _ {k}. \\ \end{array}
$$
Next, taking the full expectation from the both sides and obtain
$$
\begin{array}{l} \mathbb {E} \left[ \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \right] \leq \left(1 - \frac {\mu \gamma_ {2}}{4}\right) \mathbb {E} \left[ \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| ^ {2} \right] \\ + \frac {\sigma^ {2}}{G} \left(\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2}\right) + \rho^ {2} \left(\gamma_ {1} ^ {2} + 8 \gamma_ {2} ^ {2}\right) \mathbb {E} \mathbb {1} _ {k - \frac {1}{2}} + 2 \rho^ {2} \left(\gamma_ {2} ^ {2} + \frac {2 \gamma_ {2}}{\mu}\right) \mathbb {E} \mathbb {1} _ {k}. \\ \end{array}
$$
Unrolling the recurrence, we derive the rest of the result:
$$
\begin{array}{l} \mathbb {E} \left\| \boldsymbol {x} ^ {K + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \leq \left(1 - \frac {\mu \gamma_ {2}}{4}\right) ^ {K + 1} \left\| \boldsymbol {x} ^ {0} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {4 \sigma^ {2} \left(\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2}\right)}{\gamma_ {2} \mu (n - 2 B - m)} \\ + \rho^ {2} \left(\gamma_ {1} ^ {2} + 8 \gamma_ {2} ^ {2}\right) \sum_ {i} ^ {K} \left(1 - \frac {\mu \gamma_ {2}}{4}\right) ^ {K - i} \mathbb {E} \mathbb {1} _ {i - \frac {1}{2}} \\ + 2 \rho^ {2} \left(\gamma_ {2} ^ {2} + \frac {2 \gamma_ {2}}{\mu}\right) \sum_ {i} ^ {K} \left(1 - \frac {\mu \gamma_ {2}}{4}\right) ^ {K - i} \mathbb {E} \mathbb {1} _ {i}. \\ \end{array}
$$
Since  $\gamma_{2} \leq \frac{4}{\mu}$  and that implies
$$
\mathbb {E} \left[ \sum_ {i} ^ {T} \mathbb {1} _ {i} \left(1 - \frac {\gamma \mu}{2}\right) ^ {T - i} \right] \leq \mathbb {E} \left[ \sum_ {i} ^ {T} \mathbb {1} _ {i} \right] \leq \frac {n B}{m}. \tag {75}
$$
using the expected number of at least one peer violations (57) we derive
$$
\begin{array}{l} \mathbb {E} \left\| \boldsymbol {x} ^ {K + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \leq \left(1 - \frac {\mu \gamma_ {2}}{4}\right) ^ {K + 1} \left\| \boldsymbol {x} ^ {0} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {4 \sigma^ {2} \left(\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2}\right)}{\gamma_ {2} \mu^ {2} (n - 2 B - m)} \\ + \rho^ {2} \left(\gamma_ {1} ^ {2} + 8 \gamma_ {2} ^ {2}\right) \frac {n B}{m} + 2 \rho^ {2} \left(\gamma_ {2} ^ {2} + \frac {2 \gamma_ {2}}{\mu}\right) \frac {n B}{m} \\ \leq \left(1 - \frac {\mu \gamma_ {2}}{4}\right) ^ {K + 1} \left\| \boldsymbol {x} ^ {0} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {4 \sigma^ {2} \left(\gamma_ {1} ^ {2} + 1 6 \gamma_ {2} ^ {2}\right)}{\gamma_ {2} \mu (n - 2 B - m)} \\ + \rho^ {2} \left(\gamma_ {1} ^ {2} + 1 0 \gamma_ {2} ^ {2}\right) \frac {n B}{m} + \frac {4 \rho^ {2} \gamma_ {2}}{\mu} \frac {n B}{m}, \\ \end{array}
$$
that together with  $\rho^2 = q\sigma^2$  with  $q = 2C^2 +12 + \frac{12}{n - 2B - m}$  and  $C = \mathcal{O}(1)$  by Lemma E.1 give result of the theorem.
Corollary 9. Let assumptions of Theorem 6 hold. Then  $\mathbb{E}\left\| \pmb{x}^T - \pmb{x}^*\right\|^2 \leq \varepsilon$  holds after
$$
T = \widetilde {\mathcal {O}} \left(\frac {L}{\mu} + \frac {1}{\beta} + \frac {\sigma^ {2}}{\beta^ {2} \mu^ {2} (n - 2 B - m) \varepsilon} + \frac {q \sigma^ {2} B n}{\beta \mu^ {2} m \varepsilon} + \frac {q \sigma^ {2} B n}{\beta^ {2} \mu^ {2} m \sqrt {\varepsilon}}\right)
$$
iterations of SEG-CC with
$$
\gamma = \min \left\{\frac {1}{2 L + 2 \mu}, \frac {4 \ln \left(\max \left\{2 , \min \left\{\frac {m (n - 2 B - m) \beta^ {2} \mu^ {2} R ^ {2} K}{3 2 m \sigma^ {2} + 4 q \sigma^ {2} \beta^ {2} n B (n - 2 B - m)} , \frac {m \mu^ {2} \beta^ {2} R ^ {2} K ^ {2}}{3 2 q n B \sigma^ {2}} \right\} \right\}\right)}{\mu \beta (K + 1)} \right\}.
$$
Proof. Next, we plug  $\gamma_{2} = \beta \gamma_{1} \leq \gamma_{1} / 4$  into the result of Theorem 6 and obtain
$$
\mathbb {E} \left[ \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {x} ^ {*} \right\| ^ {2} \right] \leq \left(1 - \frac {\mu \beta \gamma_ {1}}{4}\right) \left\| \boldsymbol {x} ^ {0} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {8 \sigma^ {2} \gamma_ {1}}{\beta \mu (n - 2 B - m)} + 2 \rho^ {2} \gamma_ {1} ^ {2} \frac {n B}{m} + \frac {4 \rho^ {2} \beta \gamma_ {1}}{\mu} \frac {n B}{m}. \tag {76}
$$
Using the definition of  $\rho (\rho^2 = q\sigma^2 = \mathcal{O}(\sigma^2))$  from Lemma E.1 and if  $B\leq \frac{n}{4}$ $m <   <   n$  the result of Theorem 6 can be simplified as
$$
\mathbb {E} \left\| \boldsymbol {x} ^ {T} - \boldsymbol {x} ^ {*} \right\| ^ {2} \leq \left(1 - \frac {\mu \beta \gamma_ {1}}{4}\right) ^ {T} \left\| \boldsymbol {x} ^ {0} - \boldsymbol {x} ^ {*} \right\| ^ {2} + \frac {8 \sigma^ {2} \gamma_ {1}}{\beta \mu (n - 2 B - m)} + 2 q \sigma^ {2} \gamma_ {1} ^ {2} \frac {n B}{m} + \frac {4 q \sigma^ {2} \beta \gamma_ {1}}{\mu} \frac {n B}{m}.
$$
E.3.4 Lipschitz Monotone Case
Theorem 12. Suppose the assumptions of Theorem 11 and Assumption 5 hold. Then after  $K$  iterations of SEG-CC (Algorithm 7)
$$
\mathbb {E} \left[ \mathbf {G a p} _ {B _ {R} \left(x ^ {*}\right)} \left(\overline {{\boldsymbol {x}}} ^ {K}\right) \right] \leq \frac {3 R ^ {2}}{2 \gamma_ {2} K}, \tag {77}
$$
where  $\mathbf{G}a\mathbf{p}_{B_R(x^*)}\big(\overline{\mathbf{x}}^K\big) = \max_{u\in B_R(x^*)}\bigl \langle F(\boldsymbol {u}),\overline{\mathbf{x}}^K -\boldsymbol {u}\bigr \rangle ,\overline{\mathbf{x}}^K = \frac{1}{K}\sum_{k = 0}^{K - 1}\widetilde{\mathbf{x}}^k$  and  $R\leq \| \pmb{x}^0 -\pmb{x}^*\|$ .
Proof. We start the proof with the result of Lemma E.2.
$$
\begin{array}{l} 2 \gamma_ {2} \left\langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \widetilde {\boldsymbol {x}} ^ {k} - \boldsymbol {u} \right\rangle \leq \left\| \boldsymbol {x} ^ {k} - \boldsymbol {u} \right\| ^ {2} - \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {u} \right\| ^ {2} - 2 \gamma_ {2} \left\langle \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \boldsymbol {x} ^ {k} - \boldsymbol {u} \right\rangle \\ + 2 \gamma_ {2} ^ {2} \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) \right\| ^ {2} + 4 \gamma_ {1} \gamma_ {2} \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - F (\widetilde {\boldsymbol {x}} ^ {k}) \right\| ^ {2} \\ + 4 \gamma_ {1} \gamma_ {2} \left\| F (\boldsymbol {x} ^ {k}) - \bar {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2} + 4 \gamma_ {1} \gamma_ {2} \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2}, \\ \end{array}
$$
that leads to the following inequality
$$
\begin{array}{l} 2 \gamma_ {2} \left\langle F \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \widetilde {\boldsymbol {x}} ^ {k} - \boldsymbol {u} \right\rangle \\ \leq \left\| \boldsymbol {x} ^ {k} - \boldsymbol {u} \right\| ^ {2} - \left\| \boldsymbol {x} ^ {k + 1} - \boldsymbol {u} \right\| ^ {2} + 2 \gamma_ {2} ^ {2} \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \\ + 2 \gamma_ {2} \left\langle F (\widetilde {\boldsymbol {x}} ^ {k}) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \widetilde {\boldsymbol {x}} ^ {k} - \boldsymbol {u} \right\rangle - 2 \gamma_ {2} \left\langle \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \boldsymbol {x} ^ {k} - \boldsymbol {u} \right\rangle \\ + 4 \gamma_ {1} \gamma_ {2} \left(\left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - F (\widetilde {\boldsymbol {x}} ^ {k}) \right\| ^ {2} + \left\| F (\boldsymbol {x} ^ {k}) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2} + \left\| \overline {{\boldsymbol {g}}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2}\right). \\ \end{array}
$$
$$
\left\langle F (\boldsymbol {u}), \widetilde {\boldsymbol {x}} ^ {k} - \boldsymbol {u} \right\rangle \leq \left\langle F \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \widetilde {\boldsymbol {x}} ^ {k} - \boldsymbol {u} \right\rangle \tag {78}
$$
$$
\begin{array}{l} 2 \gamma_ {2} K \langle F (\boldsymbol {u}), \overline {{\boldsymbol {x}}} ^ {K} - \boldsymbol {u} \rangle \\ \leq \left\| \boldsymbol {x} ^ {0} - \boldsymbol {u} \right\| ^ {2} + 2 \gamma_ {2} ^ {2} \sum_ {k = 0} ^ {K - 1} \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} \\ + 2 \gamma_ {2} \sum_ {k = 0} ^ {K - 1} \left(\left\langle F (\widetilde {\boldsymbol {x}} ^ {k}) - \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \widetilde {\boldsymbol {x}} ^ {k} - \boldsymbol {u} \right\rangle - \left\langle \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \boldsymbol {x} ^ {k} - \boldsymbol {u} \right\rangle\right) \\ + 4 \gamma_ {1} \gamma_ {2} \sum_ {k = 0} ^ {K - 1} \left(\left\| \overline {{g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - F (\widetilde {\boldsymbol {x}} ^ {k}) \right\| ^ {2} + \left\| F (\boldsymbol {x} ^ {k}) - \overline {{g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2} + \left\| \overline {{g}} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) - \widehat {g} _ {\boldsymbol {\xi} ^ {k}} (\boldsymbol {x} ^ {k}) \right\| ^ {2}\right), \\ \end{array}
$$
where  $\overline{\pmb{x}}^K = \frac{1}{K}\sum_{k = 0}^{K - 1}\widetilde{\pmb{x}}^k$
Then maximization in  $\mathbf{u}$  gives
$$
\begin{array}{l} 2 \gamma_ {2} K \operatorname {G a p} _ {B _ {R} \left(x ^ {*}\right)} \left(\overline {{\boldsymbol {x}}} ^ {K}\right) \\ \leq \max  _ {u \in B _ {R} (x ^ {*})} \left\| \boldsymbol {x} ^ {0} - \boldsymbol {u} \right\| ^ {2} + 2 \gamma_ {2} ^ {2} \sum_ {k = 0} ^ {K - 1} \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) \right\| ^ {2} \\ + 4 \gamma_ {1} \gamma_ {2} \sum_ {k = 0} ^ {K - 1} \left(\left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - F \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} + \left\| F \left(\boldsymbol {x} ^ {k}\right) - \bar {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) \right\| ^ {2} + \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) \right\| ^ {2}\right) \\ + 2 \gamma_ {2} \max  _ {u \in B _ {R} (x ^ {*})} \sum_ {k = 0} ^ {K - 1} \left\langle F \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \widetilde {\boldsymbol {x}} ^ {k} - \boldsymbol {u} \right\rangle \\ + 2 \gamma_ {2} \max  _ {u \in B _ {R} (x ^ {*})} \sum_ {k = 0} ^ {K - 1} \left\langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \boldsymbol {x} ^ {k} - \boldsymbol {u} \right\rangle . \\ \end{array}
$$
By Lemma E.1
$$
2 \gamma_ {2} ^ {2} \mathbb {E} \left(\sum_ {k = 0} ^ {K - 1} \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2}\right) \leq 2 \gamma_ {2} ^ {2} \rho^ {2} \sum_ {k = 0} ^ {K - 1} \mathbb {E} [ \mathbb {1} _ {k} ] \leq 2 \gamma_ {2} ^ {2} \rho^ {2} \frac {n B}{m} \stackrel {(6 0)} {\leq} \frac {R ^ {2}}{3 2}. \tag {79}
$$
$$
4 \gamma_ {1} \gamma_ {2} \mathbb {E} \left(\sum_ {k = 0} ^ {K - 1} \left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) \right\| ^ {2}\right) \leq 4 \gamma_ {1} \gamma_ {2} \rho^ {2} \sum_ {k = 0} ^ {K - 1} \mathbb {E} \left[ \mathbb {1} _ {k - \frac {1}{2}} \right] \leq 4 \gamma_ {1} \gamma_ {2} \rho^ {2} \frac {n B}{m} \stackrel {(5 9), (6 0)} {\leq} \frac {R ^ {2}}{5} \tag {80}
$$
By Corollary 2
$$
4 \gamma_ {1} \gamma_ {2} \mathbb {E} \left(\sum_ {k = 0} ^ {K - 1} \left(\left\| \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - F \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| ^ {2} + \left\| F \left(\boldsymbol {x} ^ {k}\right) - \bar {\boldsymbol {g}} _ {\boldsymbol {\xi} ^ {k}} \left(\boldsymbol {x} ^ {k}\right) \right\| ^ {2}\right)\right) \leq \frac {8 \gamma_ {1} \gamma_ {2} K}{n - 2 B - m} \stackrel {(5 9), (6 0)} {\leq} \frac {R ^ {2}}{4}. \tag {81}
$$
$$
2 \gamma_ {2} \max  _ {u \in B _ {R} (x ^ {*})} \sum_ {k = 0} ^ {K - 1} \left\langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {k} - \boldsymbol {u} \right\rangle
$$
$$
\begin{array}{l} \leq \quad 2 \gamma_ {2} \max  _ {u \in B _ {R} (x ^ {*})} \sum_ {k = 0} ^ {K - 1} \left\langle \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\rangle \\ + 2 \gamma_ {2} \max  _ {u \in B _ {R} (x ^ {*})} \sum_ {k = 0} ^ {K - 1} \left\langle \overline {{g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \widehat {g} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \boldsymbol {x} ^ {*} - \boldsymbol {u} \right\rangle \\ \end{array}
$$
$$
\begin{array}{l} \leq \quad 2 \gamma_ {2} \sum_ {k = 0} ^ {K - 1} \left\| \boldsymbol {x} ^ {k} - \boldsymbol {x} ^ {*} \right\| \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right) \right\| \\ + 2 \gamma_ {2} \max  _ {u \in B _ {R} (x ^ {*})} \sum_ {k = 0} ^ {K - 1} \| \boldsymbol {x} ^ {*} - \boldsymbol {u} \| \left\| \widehat {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - \overline {{\boldsymbol {g}}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) \right\| \\ \end{array}
$$
$$
\begin{array}{c c} \text {L e m m a} & E. 1 \\ \leq & 6 \gamma_ {2} \rho R _ {0} \mathbb {1} _ {k}. \end{array}
$$
Taking the full expectation of both sides of the result of the previous chain we derive
$$
2 \gamma_ {2} \mathbb {E} \max  _ {u \in B _ {R} (x ^ {*})} \sum_ {k = 0} ^ {K - 1} \left\langle \overline {{g}} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}) - \widehat {g} _ {\boldsymbol {\eta} ^ {k}} (\widetilde {\boldsymbol {x}} ^ {k}), \boldsymbol {x} ^ {k} - \boldsymbol {u} \right\rangle \leq 6 \gamma_ {2} \rho R _ {0} \mathbb {E} \mathbb {1} _ {k} \leq 6 \gamma_ {2} \rho R _ {0} \frac {n B}{m} \stackrel {(6 0)} {\leq} \frac {3}{4} R ^ {2}.
$$
Now the last term
$$
2 \gamma_ {2} \max  _ {u \in B _ {R} \left(x ^ {*}\right)} \sum_ {k = 0} ^ {K - 1} \left\langle F \left(\widetilde {\boldsymbol {x}} ^ {k}\right) - \bar {\boldsymbol {g}} _ {\boldsymbol {\eta} ^ {k}} \left(\widetilde {\boldsymbol {x}} ^ {k}\right), \widetilde {\boldsymbol {x}} ^ {k} - \boldsymbol {u} \right\rangle \tag {82}
$$
Following Beznosikov et al. [2023] one can derive the bound for the next term: