SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} + (1 + \alpha^ {- 1}) \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) - F (\hat {x} _ {k}) \right. \\ \left. - \left(F _ {v _ {k - 1}} \left(\hat {x} _ {k - 1}\right) - F \left(\hat {x} _ {k - 1}\right)\right) \| ^ {2} \right] \\ \leq (1 + \alpha) L ^ {2} \mathbb {E} \left[ \| \hat {x} _ {k} - \hat {x} _ {k - 1} \| ^ {2} \right] \\ + 2 (1 + \alpha^ {- 1}) \mathbb {E} \left[ \left\| F _ {v _ {k}} (\hat {x} _ {k}) - F (\hat {x} _ {k}) \right\| ^ {2} \right. \\ + \left\| F _ {v _ {k - 1}} \left(\hat {x} _ {k - 1}\right) - F \left(\hat {x} _ {k - 1}\right) \right\| ^ {2} \bigg ] \\ \end{array}
$$
Since  $\hat{x}_0 = x_0$  and  $\hat{x}_1 = x_1 - \gamma F_{v_0}(x_0) = x_0 - (\gamma +\omega)F_{v_0}(x_0)$ , for  $k = 1$  we have
$$
\begin{array}{l} \mathbb {E} \left[ \| F _ {v _ {1}} (\hat {x} _ {1}) - F _ {v _ {0}} (\hat {x} _ {0}) \| ^ {2} \right] = \mathbb {E} \left[ \| F _ {v _ {1}} (\hat {x} _ {1}) - F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \\ \leq (1 + \alpha) \mathbb {E} \left[ \| F (\hat {x} _ {1}) - F (x _ {0}) \| ^ {2} \right] \\ + (1 + \alpha^ {- 1}) \mathbb {E} \left[ \| F _ {v _ {1}} (\hat {x} _ {1}) - F (\hat {x} _ {1}) - \left(F _ {v _ {0}} (x _ {0}) - F (x _ {0})\right) \| ^ {2} \right] \\ \leq (1 + \alpha) L ^ {2} \mathbb {E} \left[ \| \hat {x} _ {1} - x _ {0} \| ^ {2} \right] \\ + 2 (1 + \alpha^ {- 1}) \mathbb {E} \left[ \| F _ {v _ {1}} (\hat {x} _ {1}) - F (\hat {x} _ {1}) \| ^ {2} + \| F _ {v _ {0}} (x _ {0}) - F (x _ {0}) \| ^ {2} \right] \\ \end{array}
$$
Then using (8) we get,
$$
\begin{array}{l} \mathbb {E} \left[ \| F _ {v _ {1}} (\hat {x} _ {1}) - F _ {v _ {0}} (\hat {x} _ {0}) \| ^ {2} \right] \overset {(8)} {\leq} (1 + \alpha) L ^ {2} (\gamma + \omega) ^ {2} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \\ + 2 (1 + \alpha^ {- 1}) \delta \mathbb {E} \left[ \| \hat {x} _ {1} - x ^ {*} \| ^ {2} + \| x _ {0} - x ^ {*} \| ^ {2} \right] + 8 (1 + \alpha) \sigma_ {*} ^ {2} \\ \leq \left((1 + \alpha) L ^ {2} + 4 (1 + \alpha^ {- 1}) \delta\right) (\gamma + \omega) ^ {2} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \\ + 6 \left(1 + \alpha^ {- 1}\right) \delta \| x _ {0} - x ^ {*} \| ^ {2} + 8 (1 + \alpha) \sigma_ {*} ^ {2}. \\ \end{array}
$$
Let  $\{w_k\}_{k=0}^{K-1}$  be a non-increasing sequence of positive numbers that will be specified later and  $W_K = \sum_{k=0}^{K-1} w_k$ . Summing up the above two inequalities with weights  $\{w_k\}_{k=1}^{K-1}$ , we derive
$$
\begin{array}{l} \sum_ {k = 1} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) - F _ {v _ {k - 1}} (\hat {x} _ {k - 1}) \| ^ {2} \right] \leq (1 + \alpha) L ^ {2} \sum_ {k = 1} ^ {K - 3} \left(\omega (\gamma + \omega) w _ {k + 1} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] \right. \\ - \gamma \omega w _ {k + 2} \mathbb {E} \left[ \left\| F _ {v _ {k}} \left(\hat {x} _ {k}\right) \right\| ^ {2} \right]) \\ + (1 + \alpha) L ^ {2} \omega (\gamma + \omega) w _ {K - 1} \mathbb {E} \left[ \| F _ {v _ {K - 2}} (\hat {x} _ {K - 2}) \| ^ {2} \right] \\ - (1 + \alpha) L ^ {2} \gamma \omega w _ {2} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \\ + (1 + \alpha) L ^ {2} \gamma (\gamma + \omega) \sum_ {k = 1} ^ {K - 2} w _ {k + 1} \mathbb {E} [ \| F _ {v _ {k}} (\hat {x} _ {k}) \\ \left. - F _ {v _ {k - 1}} \left(\hat {x} _ {k - 1}\right) \| ^ {2} \right] + 4 \left(1 + \alpha^ {- 1}\right) \delta \sum_ {k = 2} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| x _ {k} - x ^ {*} \| ^ {2} \right] \\ + w _ {k} \mathbb {E} \left[ \| x _ {k - 1} - x ^ {*} \| ^ {2} \right] \\ + 4 (1 + \alpha^ {- 1}) \delta \gamma^ {2} \sum_ {k = 1} ^ {K - 2} w _ {k + 1} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right. \\ \left. + \left\| F _ {v _ {k - 1}} \left(\hat {x} _ {k - 1}\right) \right\| ^ {2} \right] + 8 \left(1 + \alpha^ {- 1}\right) \left(W _ {K} - w _ {0} - w _ {1}\right) \sigma_ {*} ^ {2} \\ + \left((1 + \alpha) L ^ {2} + 4 (1 + \alpha^ {- 1}) \delta\right) (\gamma + \omega) ^ {2} w _ {1} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \\ + 6 \left(1 + \alpha^ {- 1}\right) \delta w _ {1} \left\| x _ {0} - x ^ {*} \right\| ^ {2} + 8 (1 + \alpha) w _ {1} \sigma_ {*} ^ {2}. \\ \end{array}
$$
Next, we rearrange the terms using  $w_{k} \geq w_{k + 1}$  and new notation  $\Delta_k = \mathbb{E}\left[\| F_{v_k}(\hat{x}_k) - F_{v_{k - 1}}(\hat{x}_{k - 1})\| ^2\right]$ :
$$
\begin{array}{l} \left(1 - (1 + \alpha) L ^ {2} \gamma (\gamma + \omega)\right) \sum_ {k = 1} ^ {K - 1} w _ {k} \Delta_ {k} \leq \sum_ {k = 1} ^ {K - 2} (1 + \alpha) L ^ {2} \omega (\gamma + \omega) w _ {k} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] \\ + 8 \left(1 + \alpha^ {- 1}\right) \delta \gamma^ {2} w _ {k} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] \\ + \left((1 + \alpha) L ^ {2} + 8 (1 + \alpha^ {- 1}) \delta\right) (\gamma + \omega) ^ {2} w _ {0} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \\ + 1 2 (1 + \alpha^ {- 1}) \delta \sum_ {k = 1} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| x _ {k} - x ^ {*} \| ^ {2} \right] \\ + 8 \left(1 + \alpha^ {- 1}\right) \left(W _ {K} - w _ {0}\right) \sigma_ {*} ^ {2}. \\ \end{array}
$$
To simplify the above inequality we choose  $\alpha = \frac{1}{2L^2\gamma(\gamma + \omega)} -\frac{1}{2}$ , which is positive due to  $\gamma < 1 / L$  and  $\gamma +\omega < 1 / L$ . In this case, we have
$$
(1 + \alpha) L ^ {2} \gamma (\gamma + \omega) = \frac {1}{2} L ^ {2} \gamma (\gamma + \omega) + \frac {1}{2},
$$
$$
(1 + \alpha) L ^ {2} (\gamma + \omega) ^ {2} = \frac {1}{2} L ^ {2} (\gamma + \omega) ^ {2} + \frac {\gamma + \omega}{2 \gamma} \leq \frac {3}{2},
$$
$$
(1 + \alpha) L ^ {2} \omega (\gamma + \omega) = \frac {1}{2} L ^ {2} \omega (\gamma + \omega) + \frac {\omega}{2 \gamma} = \frac {L \omega}{2} \left(L (\gamma + \omega) + \frac {1}{\gamma L}\right) \leq \frac {3 L \omega}{2},
$$
$$
{1 + \alpha^ {- 1}} = {1 + \frac {2 L ^ {2} \gamma (\gamma + \omega)}{1 - L ^ {2} \gamma (\gamma + \omega)} = \frac {1 + L ^ {2} \gamma (\gamma + \omega)}{1 - L ^ {2} \gamma (\gamma + \omega)} \leq \frac {2}{1 - L ^ {2} \gamma (\gamma + \omega)},}
$$
where we also use  $\frac{1}{2L} < \gamma < \frac{1}{L}$  and  $\gamma + \omega < \frac{1}{L}$ . Using these relations, we can continue our derivation as follows:
$$
\begin{array}{l} \frac {1}{2} \left(1 - L ^ {2} \gamma (\gamma + \omega)\right) \sum_ {k = 1} ^ {K - 1} w _ {k} \Delta_ {k} \leq \sum_ {k = 1} ^ {K - 2} \left(\frac {3 L \omega}{2} + \frac {1 6}{1 - L ^ {2} \gamma (\gamma + \omega)} \delta \gamma^ {2}\right) w _ {k} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] \\ \left. + \left(\frac {3}{2} + \frac {1 6}{1 - L ^ {2} \gamma (\gamma + \omega)} \delta (\gamma + \omega) ^ {2}\right) w _ {0} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \right. \\ + \frac {2 4}{1 - L ^ {2} \gamma (\gamma + \omega)} \delta \sum_ {k = 1} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| x _ {k} - x ^ {*} \| ^ {2} \right] \\ + \frac {1 6}{1 - L ^ {2} \gamma (\gamma + \omega)} (W _ {K} - w _ {0}) \sigma_ {*} ^ {2}. \\ \end{array}
$$
Dividing both sides by  $\frac{1}{2}\left(1 - L^2\gamma (\gamma +\omega)\right)$ , we derive
$$
\begin{array}{l} \sum_ {k = 1} ^ {K - 1} w _ {k} \Delta_ {k} \leq \sum_ {k = 1} ^ {K - 2} \left(\frac {3 L \omega}{1 - L ^ {2} \gamma (\gamma + \omega)} + \frac {3 2}{(1 - L ^ {2} \gamma (\gamma + \omega)) ^ {2}} \delta \gamma^ {2}\right) w _ {k} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] \\ + \left(\frac {3}{1 - L ^ {2} \gamma (\gamma + \omega)} + \frac {3 2}{(1 - L ^ {2} \gamma (\gamma + \omega)) ^ {2}} \delta (\gamma + \omega) ^ {2}\right) w _ {0} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \\ + \frac {4 8}{(1 - L ^ {2} \gamma (\gamma + \omega)) ^ {2}} \delta \sum_ {k = 1} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| x _ {k} - x ^ {*} \| ^ {2} \right] \\ + \frac {3 2}{(1 - L ^ {2} \gamma (\gamma + \omega)) ^ {2}} (W _ {K} - w _ {0}) \sigma_ {*} ^ {2} \\ = \sum_ {k = 1} ^ {K - 2} C _ {1} w _ {k} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] + C _ {2} w _ {0} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \\ + 3 C _ {3} \delta \sum_ {k = 1} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| x _ {k} - x ^ {*} \| ^ {2} \right] + 2 C _ {3} W _ {K} \sigma_ {*} ^ {2}, \tag {41} \\ \end{array}
$$
where  $C_1 = \frac{3L\omega}{1 - L^2\gamma(\gamma + \omega)} + \frac{32}{(1 - L^2\gamma(\gamma + \omega))^2}\delta \gamma^2$ ,  $C_2 = \frac{3}{1 - L^2\gamma(\gamma + \omega)} + \frac{32}{(1 - L^2\gamma(\gamma + \omega))^2}\delta (\gamma + \omega)^2$  and  $C_3 = \frac{16}{(1 - L^2\gamma(\gamma + \omega))^2}$ . Summing up inequalities (39) for  $k = 1, \ldots, K - 1$  with weights  $w_1, \ldots, w_{K-1}$  and (40) with weight  $w_0$ , we get
$$
\begin{array}{l} \sum_ {k = 0} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| x _ {k + 1} - x ^ {*} \| ^ {2} \right] \leq \sum_ {k = 0} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| x _ {k} - x ^ {*} \| ^ {2} \right] - \omega \gamma \sum_ {k = 1} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| F _ {v _ {k - 1}} (\hat {x} _ {k - 1}) \| ^ {2} \right] \\ + \omega \gamma \sum_ {k = 1} ^ {K - 1} w _ {k} \Delta_ {k} + \omega (\omega + 2 \rho) w _ {0} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right]. \\ \end{array}
$$
Since  $w_{k}\geq w_{k + 1}$  , we can continue the derivation as follows:
$$
\begin{array}{l} \sum_ {k = 0} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| x _ {k + 1} - x ^ {*} \| ^ {2} \right] \leq \sum_ {k = 0} ^ {K - 1} w _ {k} \mathbb {E} \left[ \| x _ {k} - x ^ {*} \| ^ {2} \right] - \omega \gamma \sum_ {k = 0} ^ {K - 2} w _ {k} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] \\ + \omega \gamma \sum_ {k = 1} ^ {K - 1} w _ {k} \Delta_ {k} + \omega (\omega + 2 \rho) w _ {0} \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \\ \stackrel {(4 1)} {\leq} \sum_ {k = 0} ^ {K - 1} (1 + 3 C _ {3} \omega \gamma \delta) w _ {k} \mathbb {E} \left[ \| x _ {k} - x ^ {*} \| ^ {2} \right] \\ - \omega \gamma (1 - C _ {1}) \sum_ {k = 0} ^ {K - 2} w _ {k} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] \\ + 2 \omega \gamma C _ {2} w _ {0} \mathbb {E} \left[ \| F _ {v _ {0}} (\hat {x} _ {0}) \| ^ {2} \right] + 2 \omega \gamma C _ {3} W _ {K} \sigma_ {*} ^ {2}. \\ \end{array}
$$
Now we need to specify the weights  $w_{-1}, w_0, w_1, \ldots, w_{K - 1}$ . Let  $w_{K - 2} = 1$  and  $w_{k - 1} = (1 + 3C_3\omega \gamma \delta)w_k$ . Then, rearranging the terms, dividing both sides by  $\omega \gamma (1 - C_1)W_{K - 1}$ , we get
$$
\begin{array}{l} \min _ {0 \leq k \leq K - 1} \mathbb {E} \left[ \| F (\hat {x} _ {k}) \| ^ {2} \right] \leq \min _ {0 \leq k \leq K - 1} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] \\ \leq \sum_ {k = 0} ^ {K - 2} \frac {w _ {k}}{W _ {K - 1}} \mathbb {E} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] \\ \leq \frac {1}{\omega \gamma (1 - C _ {1}) W _ {K - 1}} \sum_ {k = 0} ^ {K - 1} \left(w _ {k - 1} \mathbb {E} \left[ \| x _ {k} - x ^ {*} \| ^ {2} \right] \right. \\ \left. - w _ {k} \mathbb {E} \left[ \left\| x _ {k + 1} - x ^ {*} \right\| ^ {2} \right]\right) + \frac {2 C _ {2} w _ {0} \mathbb {E} \left[ \left\| F _ {v _ {0}} (\hat {x} _ {0}) \right\| ^ {2} \right]}{\left(1 - C _ {1}\right) W _ {K - 1}} \\ + \frac {2 C _ {3} W _ {K} \sigma_ {*} ^ {2}}{\left(1 - C _ {1}\right) W _ {K - 1}} \\ \leq \frac {w _ {- 1} \| x _ {0} - x ^ {*} \| ^ {2}}{\omega \gamma (1 - C _ {1}) W _ {K - 1}} + \frac {2 C _ {2} w _ {0} \mathbb {E} \left[ \| F _ {v _ {0}} (\hat {x} _ {0}) \| ^ {2} \right]}{(1 - C _ {1}) W _ {K - 1}} + \frac {2 C _ {3} W _ {K} \sigma_ {*} ^ {2}}{(1 - C _ {1}) W _ {K - 1}}. \\ \end{array}
$$
It remains to simplify the right-hand side of the above inequality. First, we notice that  $W_{K - 1} = \sum_{k = 0}^{K - 2}w_k\geq (K - 1)w_{K - 2} = K - 1$  since  $w_{k}\geq w_{k + 1}$ . Moreover,  $w_{-1} = (1 + 3C_3\omega \gamma \delta)^{K - 1}$ . Next,
$$
\begin{array}{l} {C _ {1}} = {\frac {3 L \omega}{1 - L ^ {2} \gamma (\gamma + \omega)} + \frac {3 2}{(1 - L ^ {2} \gamma (\gamma + \omega)) ^ {2}} \delta \gamma^ {2}} \\ \leq \frac {3 L \omega}{1 - L \gamma} + \frac {3 2}{(1 - L \gamma) ^ {2}} \cdot \frac {(1 - L \gamma) L ^ {3} \omega}{3 2} \cdot \gamma^ {2} \leq \frac {4 L \omega}{1 - L \gamma}, \\ \end{array}
$$
$$
\begin{array}{l} {C _ {2}} = {\frac {3}{1 - L ^ {2} \gamma (\gamma + \omega)} + \frac {3 2}{(1 - L ^ {2} \gamma (\gamma + \omega)) ^ {2}} \delta (\gamma + \omega) ^ {2}} \\ \leq \frac {3}{1 - L \gamma} + \frac {3 2}{(1 - L \gamma) ^ {2}} \cdot \frac {(1 - L \gamma) L ^ {3} \omega}{3 2} \cdot (\gamma + \omega) ^ {2} \leq \frac {4}{1 - L \gamma}, \\ \end{array}
$$
$$
C _ {3} = \frac {1 6}{(1 - L ^ {2} \gamma (\gamma + \omega)) ^ {2}} \leq \frac {1 6}{(1 - L \gamma) ^ {2}},
$$
where we use  $\delta \leq (1 - L\gamma)L^3\omega /16$  and  $\gamma +\omega <  1 / L$  . Using these inequalities, we simplify the bound as follows:
$$
\begin{array}{l} \min _ {0 \leq k \leq K - 1} \mathbb {E} \left[ \| F (\hat {x} _ {k}) \| ^ {2} \right] \leq \frac {(1 - L \gamma) (1 + 3 C _ {3} \omega \gamma \delta) ^ {K - 1} \| x _ {0} - x ^ {*} \| ^ {2}}{\omega \gamma (1 - L (\gamma + 4 \omega)) (K - 1)} \\ + \frac {8 (1 + 3 C _ {3} \omega \gamma \delta) ^ {K - 2} \mathbb {E} \left[ \| F _ {v _ {0}} (\hat {x} _ {0}) \| ^ {2} \right]}{(1 - L (\gamma + 4 \omega)) (K - 1)} \\ + \frac {3 2 \sigma_ {*} ^ {2}}{(1 - L \gamma) (1 - L (\gamma + 4 \omega))} \\ \leq \frac {(1 - L \gamma) \left(1 + \frac {4 8 \omega \gamma \delta}{(1 - L \gamma) ^ {2}}\right) ^ {K - 1} \| x _ {0} - x ^ {*} \| ^ {2}}{\omega \gamma (1 - L (\gamma + 4 \omega)) (K - 1)} \\ + \frac {8 \left(1 + \frac {4 8 \omega \gamma \delta}{(1 - L \gamma) ^ {2}}\right) ^ {K - 2} \mathbb {E} \left[ \| F _ {v _ {0}} (\hat {x} _ {0}) \| ^ {2} \right]}{(1 - L (\gamma + 4 \omega)) (K - 1)} \\ + \frac {3 2 \sigma_ {*} ^ {2}}{(1 - L \gamma) (1 - L (\gamma + 4 \omega))} \tag {42} \\ \end{array}
$$
where we use  $W_{K} = W_{K - 1} + w_{K - 1} \leq W_{K - 1} + w_{K - 2} \leq 2W_{K - 1}$ . Finally, we use (8) to upper-bound  $\mathbb{E}\left[\| F_{v_0}(\hat{x}_0)\|^2\right]$ :
$$
\begin{array}{l} \mathbb {E} \left[ \| F _ {v _ {0}} (\hat {x} _ {0}) \| ^ {2} \right] = \mathbb {E} \left[ \| F _ {v _ {0}} (x _ {0}) \| ^ {2} \right] \overset {(8)} {\leq} \delta \| x _ {0} - x ^ {*} \| ^ {2} + \| F (x _ {0}) \| ^ {2} + 2 \sigma_ {*} ^ {2} \\ \leq \left(\delta + L ^ {2}\right) \left\| x _ {0} - x ^ {*} \right\| ^ {2} + 2 \sigma_ {*} ^ {2}. \\ \end{array}
$$
Plugging this inequality in (42), we derive
$$
\begin{array}{l} \min  _ {0 \leq k \leq K - 1} \mathbb {E} \left[ \| F (\hat {x} _ {k}) \| ^ {2} \right] \leq \frac {\left(1 + 8 \omega \gamma (\delta + L ^ {2}) - L \gamma\right) \left(1 + \frac {4 8 \omega \gamma \delta}{(1 - L \gamma) ^ {2}}\right) ^ {K - 1} \| x _ {0} - x ^ {*} \| ^ {2}}{\omega \gamma (1 - L (\gamma + 4 \omega)) (K - 1)} \\ + \frac {4 \left(8 + \frac {1 - L \gamma}{K - 1} \left(1 + \frac {4 8 \omega \gamma \delta}{(1 - L \gamma) ^ {2}}\right) ^ {K - 1}\right) \sigma_ {*} ^ {2}}{(1 - L \gamma) (1 - L (\gamma + 4 \omega))}, \\ \end{array}
$$
E.5 Proof of Theorem 4.5
Theorem E.5. Let  $F$  be  $L$ -Lipschitz and satisfy Weak Minty condition with parameter  $\rho < 1/(2L)$ . Assume that inequality (8) holds (e.g., it holds whenever Assumption 3.1 holds, see Lemma 3.2). Assume that  $\gamma_k = \gamma$ ,  $\omega_k = \omega$  and
$$
\max  \left\{2 \rho , \frac {1}{2 L} \right\} <   \gamma <   \frac {1}{L}, \quad 0 <   \omega <   \min  \left\{\gamma - 2 \rho , \frac {1}{4 L} - \frac {\gamma}{4} \right\}.
$$
Then, for all  $K \geq 2$  the iterates produced by mini-batched SPEG with batch-size
$$
\tau \geq \max  \left\{1, \frac {3 2 \delta}{(1 - L \gamma) L ^ {3} \omega}, \frac {4 8 \omega \gamma \delta (K - 1)}{(1 - L \gamma) ^ {2}}, \frac {2 \omega \gamma \sigma_ {*} ^ {2} (K - 1)}{(1 - L \gamma) \| x _ {0} - x ^ {*} \| ^ {2}} \right\} \tag {43}
$$
$$
\min  _ {0 \leq k \leq K - 1} \mathbb {E} \left[ \| F (\hat {x} _ {k}) \| ^ {2} \right] \leq \frac {4 8 \| x _ {0} - x ^ {*} \| ^ {2}}{\omega \gamma (1 - L (\gamma + 4 \omega)) (K - 1)}. \tag {44}
$$
Proof. Mini-batched SPEG uses estimator
$$
F _ {v _ {k}} (\hat {x} _ {k}) = \frac {1}{\tau} \sum_ {i = 1} ^ {\tau} F _ {v _ {k, i}} (\hat {x} _ {k}),
$$
where  $F_{v_{k,1}}(\hat{x}_k),\dots ,F_{v_{k,\tau}}(\hat{x}_k)$  are independent samples satisfying (8) with parameters  $\delta$  and  $\sigma_*^2$ . Using variance decomposition and independence of  $F_{v_{k,1}}(\hat{x}_k),\dots ,F_{v_{k,\tau}}(\hat{x}_k)$ , we get
$$
\begin{array}{l} \mathbb {E} _ {v _ {k}} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) \| ^ {2} \right] = \mathbb {E} _ {v _ {k}} \left[ \| F _ {v _ {k}} (\hat {x} _ {k}) - F (\hat {x} _ {k}) \| ^ {2} \right] + \| F (\hat {x} _ {k}) \| ^ {2} \\ { = } { \mathbb { E } _ { v _ { k } } \left[ \left\| \frac { 1 } { \tau } \sum _ { i = 1 } ^ { b } ( F _ { v _ { k , i } } ( \hat { x } _ { k } ) - F ( \hat { x } _ { k } ) ) \right\| ^ { 2 } \right] + \| F ( \hat { x } _ { k } ) \| ^ { 2 } } \\ { = } { \frac { 1 } { \tau ^ { 2 } } \sum _ { i = 1 } ^ { \tau } \mathbb { E } _ { v _ { k } } \left[ \| F _ { v _ { k , i } } ( \hat { x } _ { k } ) - F ( \hat { x } _ { k } ) \| ^ { 2 } \right] + \| F ( \hat { x } _ { k } ) \| ^ { 2 } } \\ \stackrel {(8)} {\leq} \frac {\delta}{\tau} \| \hat {x} _ {k} - x ^ {*} \| ^ {2} + \| F (\hat {x} _ {k}) \| ^ {2} + \frac {2 \sigma_ {*} ^ {2}}{\tau}. \\ \end{array}
$$
That is, mini-batched estimator  $F_{v_k}(\hat{x}_k)$  satisfies (8) with parameters  $\delta/\tau$  and  $\sigma_{*}^{2}/\tau$ . Therefore, Theorem E.4 implies
$$
\begin{array}{l} \min _ {0 \leq k \leq K - 1} \mathbb {E} \left[ \| F (\hat {x} _ {k}) \| ^ {2} \right] \leq \frac {(1 + 4 \omega \gamma (\frac {\delta}{\tau} + L ^ {2}) - L \gamma) (1 + \frac {4 8 \omega \gamma \delta}{(1 - L \gamma) ^ {2} \tau}) ^ {K - 1} \| x _ {0} - x ^ {*} \| ^ {2}}{\omega \gamma (1 - L (\gamma + 4 \omega)) (K - 1)} \\ + \frac {8 \left(8 + \frac {1 - L \gamma}{K - 1} \left(1 + \frac {4 8 \omega \gamma \delta}{\left(1 - L \gamma\right) ^ {2} \tau}\right) ^ {K - 1}\right) \sigma_ {*} ^ {2}}{\left(1 - L \gamma\right) \left(1 - L (\gamma + 4 \omega)\right) \tau}. \tag {45} \\ \end{array}
$$
Since  $\tau$  satisfies (43) and  $\gamma \leq 1 / L$ ,  $\omega \leq 1 / 4L$ , we have
$$
4 \omega \gamma \left(\frac {\delta}{\tau} + L ^ {2}\right) \leq \frac {1}{4 L ^ {2}} \left(\delta \cdot \frac {(1 - L \gamma) L ^ {3} \omega}{1 6 \delta} + L ^ {2}\right) \leq 1,
$$
$$
\begin{array}{l} \left(1 + \frac {4 8 \omega \gamma \delta}{(1 - L \gamma) ^ {2} \tau}\right) ^ {K - 1} \leq \left(1 + \frac {4 8 \omega \gamma \delta}{(1 - L \gamma) ^ {2}} \cdot \frac {(1 - L \gamma) ^ {2}}{4 8 \omega \gamma \delta (K - 1)}\right) ^ {K - 1} \\ = \left(1 + \frac {1}{K - 1}\right) ^ {K - 1} \leq \exp (1) <   3. \\ \end{array}
$$
Using this, we can simplify (45) as follows:
$$
\begin{array}{l} \min  _ {0 \leq k \leq K - 1} \mathbb {E} \left[ \| F (\hat {x} _ {k}) \| ^ {2} \right] \leq \frac {6 \| x _ {0} - x ^ {*} \| ^ {2}}{\omega \gamma (1 - L (\gamma + 4 \omega)) (K - 1)} + \frac {8 8 \sigma_ {*} ^ {2}}{(1 - L \gamma) (1 - L (\gamma + 4 \omega)) \tau} \\ \stackrel {(4 3)} {\leq} \frac {6 \| x _ {0} - x ^ {*} \| ^ {2}}{\omega \gamma (1 - L (\gamma + 4 \omega)) (K - 1)} \\ + \frac {8 8 \sigma_ {*} ^ {2}}{(1 - L \gamma) (1 - L (\gamma + 4 \omega))} \cdot \frac {(1 - L \gamma) \| x _ {0} - x ^ {*} \| ^ {2}}{2 \omega \gamma \sigma_ {*} ^ {2}} \\ = \frac {4 8 \| x _ {0} - x ^ {*} \| ^ {2}}{\omega \gamma (1 - L (\gamma + 4 \omega)) (K - 1)}. \\ \end{array}
$$
On Oracle Complexity of Theorem 4.5. Let us now express the result of Theorem 4.5 via oracle complexity.
Oracle complexity captures the computational requirements required to solve a specific optimization problem. That is, given a prespecified accuracy  $\varepsilon > 0$ , it measures the number of oracle calls needed to solve the problem to this  $\varepsilon$  accuracy. In our setting, an oracle call indicates the computation of one operator,  $F_{i}$  (for some  $i \in [n]$ ). Therefore, in Theorem 4.5, where a mini-batch of size  $\tau$  is required in each iteration of the update rule, we have  $\tau$  many oracle calls per iteration. In that scenario, the total number of oracle calls required to obtain specific accuracy  $\varepsilon > 0$  is given by  $K\tau$  (multiplication of  $K$  iterations with  $\tau$  oracle calls).
Note that according to Theorem 4.5 to achieve an  $\varepsilon$  accuracy, we need  $K \geq \frac{C\|x_0 - x^*\|^2}{\epsilon}$  iterations. This follows trivially by
$$
\min  _ {0 \leq k \leq K - 1} \mathbb {E} \left[ \| F (\hat {x} _ {k}) \| ^ {2} \right] \overset {\text {T h e o r e m}} {\leq} ^ {4. 5} \frac {C \| x _ {0} - x ^ {*} \| ^ {2}}{K - 1} \leq \varepsilon . \tag {46}
$$
Therefore, using  $K \geq \frac{C\|x_0 - x^*\|^2}{\epsilon}$  in combination with the lower bound on  $\tau$  from (15), the total number of oracle calls to satisfy (46) is given by:
$$
K \tau \geq \max \left\{\frac {C | | x _ {0} - x ^ {*} | | ^ {2}}{\epsilon}, \frac {3 2 C \delta | | x _ {0} - x ^ {*} | | ^ {2}}{(1 - L \gamma) L ^ {3} \omega \epsilon}, \frac {4 8 C ^ {2} \omega \gamma \delta | | x _ {0} - x ^ {*} | | ^ {4}}{(1 - \gamma L) ^ {2} \epsilon^ {2}}, \frac {2 C ^ {2} \omega \gamma \sigma_ {*} ^ {2} | | x _ {0} - x ^ {*} | | ^ {2}}{(1 - L \gamma) \epsilon^ {2}} \right\}.
$$
F Further Results on Arbitrary Sampling
Expanding the left hand side of Expected Residual (ER) condition we have
$$
\begin{array}{l} \mathbb {E} \| (F _ {v} (x) - F _ {v} (x ^ {*})) - (F (x) - F (x ^ {*})) \| ^ {2} \stackrel {(2 5)} {=} \mathbb {E} \| (F _ {v} (x) - F _ {v} (x ^ {*})) \| ^ {2} - \| F (x) - F (x ^ {*}) \| ^ {2} \\ \leq \mathbb {E} \| F _ {v} (x) - F _ {v} \left(x ^ {*}\right) \| ^ {2}. \tag {47} \\ \end{array}
$$
For any  $x$  and  $y$  with  $v_{i} = \frac{1}{p_{i}}$  we obtain
$$
\begin{array}{l} {\| F _ {v} (x) - F _ {v} (y) \| ^ {2}} = {\frac {1}{n ^ {2}} \bigg \| \sum_ {i \in S} \frac {1}{p _ {i}} (F _ {i} (x) - F _ {i} (y)) \bigg \| ^ {2}} \\ = \sum_ {i, j \in S} \left\langle \frac {1}{n p _ {i}} (F _ {i} (x) - F _ {i} (y)), \frac {1}{n p _ {j}} (F _ {j} (x) - F _ {j} (y)) \right\rangle . \\ \end{array}
$$
Then taking expectation on both sides we get
$$
\begin{array}{l} \mathbb {E} \| F _ {v} (x) - F _ {v} (y) \| ^ {2} = \sum_ {C} p _ {C} \sum_ {i, j \in C} \left\langle \frac {1}{n p _ {i}} (F _ {i} (x) - F _ {i} (y)), \frac {1}{n p _ {j}} (F _ {j} (x) - F _ {j} (y)) \right\rangle \\ = \sum_ {i, j = 1} ^ {n} \sum_ {C: i, j \in C} p _ {C} \left\langle \frac {1}{n p _ {i}} (F _ {i} (x) - F _ {i} (y)), \frac {1}{n p _ {j}} (F _ {j} (x) - F _ {j} (y)) \right\rangle \\ = \sum_ {i, j = 1} ^ {n} \frac {P _ {i j}}{p _ {i} p _ {j}} \Bigg \langle \frac {1}{n} (F _ {i} (x) - F _ {i} (y)), \frac {1}{n} (F _ {j} (x) - F _ {j} (y)) \Bigg \rangle . \\ \end{array}
$$
Now we consider the case, where the ratio  $\frac{P_{ij}}{p_i p_j} = c_2$  i.e. constant for  $i \neq j$  and  $P_{ii} = p_i$ . Then from the above computations we derive
$$
\mathbb {E} \| F _ {v} (x) - F _ {v} (y) \| ^ {2} = \sum_ {i \neq j} ^ {n} c _ {2} \Bigg \langle \frac {1}{n} (F _ {i} (x) - F _ {i} (y)), \frac {1}{n} (F _ {i} (x) - F _ {i} (y)) \Bigg \rangle
$$
$$
\begin{array}{l} + \sum_ {i = 1} ^ {n} \frac {1}{n ^ {2} p _ {i}} \| F _ {i} (x) - F _ {i} (y) \| ^ {2} \\ = \sum_ {i, j = 1} ^ {n} c _ {2} \left\langle \frac {1}{n} \left(F _ {i} (x) - F _ {i} (y)\right), \frac {1}{n} \left(F _ {i} (x) - F _ {i} (y)\right) \right\rangle \\ + \sum_ {i = 1} ^ {n} \frac {1 - p _ {i} c _ {2}}{n ^ {2} p _ {i}} \| F _ {i} (x) - F _ {i} (y) \| ^ {2} \\ \end{array}
$$
$$
\stackrel {(1 9)} {\leq} c _ {2} \| F (x) - F (y) \| ^ {2} + \sum_ {i = 1} ^ {n} \frac {1 - p _ {i} c _ {2}}{n ^ {2} p _ {i}} L _ {i} ^ {2} \| x - y \| ^ {2}
$$
$$
\stackrel {(1 8)} {\leq} \left(c _ {2} L ^ {2} + \frac {1}{n ^ {2}} \sum_ {i = 1} ^ {n} \frac {1 - p _ {i} c _ {2}}{p _ {i}} L _ {i} ^ {2}\right) \| x - y \| ^ {2}.
$$
Thus replacing  $y = x^{*}$  and combining with (47) we get the following bound on the Expected Residual: