SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \left| \left| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \\ \stackrel {(A. 1 7)} {\leq} (1 - \eta_ {t} \mu) \| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \| ^ {2} - 2 \eta_ {t} \left(f _ {i _ {t}} (\mathbf {x} _ {t}) - f _ {i _ {t}} (\mathbf {x} ^ {\star})\right) + \eta_ {t} \frac {\gamma_ {t} \| \nabla f _ {i _ {t}} (\mathbf {x} _ {t}) \| ^ {2}}{c _ {l} \sqrt {\sum_ {s = 0} ^ {t} \gamma_ {s} \| \nabla f _ {i _ {s}} (\mathbf {x} _ {s}) \| ^ {2}}} \\ \leq \left(1 - \eta_ {t} \mu\right) \left\| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\| ^ {2} + 2 \eta_ {t} \left(f _ {i _ {t}} \left(\mathbf {x} ^ {\star}\right) - f _ {i _ {t}} ^ {\star}\right) - 2 \eta_ {t} \left(f _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - f _ {i _ {t}} ^ {\star}\right) + \eta_ {t} \frac {f _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - f _ {i _ {t}} ^ {\star}}{c _ {l} \rho \sqrt {\sum_ {s = 0} ^ {t} \gamma_ {s} \left\| \left\| \nabla f _ {i _ {s}} \left(\mathbf {x} _ {s}\right) \right\| ^ {2} \right.}} \\ \leq \left(1 - \eta_ {t} \mu\right)\left\| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\| ^ {2} + 2 \eta_ {t} \sigma_ {\max } ^ {2} - 2 \eta_ {t} \underbrace {\left(f _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - f _ {i _ {t}} ^ {\star}\right)} _ {\geq 0} + \eta_ {t} \frac {f _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - f _ {i _ {t}} ^ {\star}}{c _ {l} \rho \sqrt {\sum_ {s = 0} ^ {t} \gamma_ {s} \left\| \right.\left\| \nabla f _ {i _ {s}} \left(\mathbf {x} _ {s}\right)\right\| ^ {2}}} , \tag {B.22} \\ \end{array}
$$
where  $\sigma_{\mathrm{max}}^2 = \max_{i_t}\left\{f_{i_t}(\mathbf{x}^\star) - f_{i_t}^\star \right\}$ . We can then compare  $c_l\rho \sqrt{\sum_{s=0}^{t}\gamma_s||\nabla f_{i_s}(\mathbf{x}_s)||^2}$  with  $\frac{1}{2}$  and apply Lemma 13 correspondingly.
B.3 Proofs for Theorem 2 and 6
Proof. For clarity, we separate the proofs for AdaSPS and AdaSLS.
AdaSPS: Plugging in the upper bound of  $\eta_t$  in Lemma 15, we have:
$$
\left\| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right\| ^ {2} \stackrel {(A. 1 6)} {\leq} \left\| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\| ^ {2} - 2 \eta_ {t} \left\langle \nabla f _ {i _ {t}} (\mathbf {x} _ {t}), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\rangle + \eta_ {t} \frac {f _ {i _ {t}} (\mathbf {x} _ {t}) - f ^ {\star}}{c _ {p} \sqrt {\sum_ {s = 0} ^ {t} f _ {i _ {s}} (\mathbf {x} _ {s}) - f ^ {\star}}} . \tag {B.23}
$$
Since  $c_{p}\sqrt{f_{i_{0}}(\mathbf{x}_{0}) - f^{\star}}\geq 1$  (B.23) can be reduced to:
$$
\left| \left| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \leq \left| \left| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right| \right| ^ {2} - 2 \eta_ {t} \left\langle \nabla f _ {i _ {t}} \left(\mathbf {x} _ {t}\right), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\rangle + \eta_ {t} \left(f _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - f ^ {\star}\right). \tag {B.24}
$$
By convexity of  $f_{i_t}$ , we get:
$$
\left\| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right\| ^ {2} \leq \left\| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\| ^ {2} - \eta_ {t} \left\langle \nabla f _ {i t} \left(\mathbf {x} _ {t}\right), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\rangle . \tag {B.25}
$$
Note that  $\langle \nabla f_{i_t}(\mathbf{x}_t),\mathbf{x}_t - \mathbf{x}^\star \rangle \geq 0$  and  $\eta_t$  is non-increasing, we thus get:
$$
\left| \left| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \leq \left| \left| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right| \right| ^ {2} - \eta_ {T - 1} \left\langle \nabla f _ {i _ {t}} \left(\mathbf {x} _ {t}\right), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\rangle . \tag {B.26}
$$
We first show that  $\eta_{T - 1}$  is always lower bounded. From equation (B.25) and using convexity of  $f_{i_t}$ , we get:
$$
\eta_ {t} \left(f _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - f ^ {\star}\right) \leq \left| \left| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right| \right| ^ {2} - \left| \left| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right| \right| ^ {2}. \tag {B.27}
$$
$$
\sum_ {t = 0} ^ {T - 1} \eta_ {t} \left(f _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - f ^ {\star}\right) \leq \left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\| ^ {2}. \tag {B.28}
$$
Using the lower bound of  $\eta_t$ , we get:
$$
\frac {1}{2 c _ {p} L} \sqrt {\sum_ {s = 0} ^ {T - 1} f _ {i _ {s}} (\mathbf {x} _ {s}) - f ^ {\star}} \stackrel {(A. 6)} {\leq} \frac {1}{2 c _ {p} L} \sum_ {t = 0} ^ {T - 1} \frac {f _ {i _ {t}} (\mathbf {x} _ {t}) - f ^ {\star}}{\sqrt {\sum_ {s = 0} ^ {t} f _ {i _ {s}} (\mathbf {x} _ {s}) - f ^ {\star}}} \stackrel {(A. 1 6)} {\leq} \sum_ {t = 0} ^ {T - 1} \eta_ {t} \left(f _ {i _ {t}} (\mathbf {x} _ {t}) - f ^ {\star}\right), \tag {B.29}
$$
This reveals that:
$$
\eta_ {T - 1} \stackrel {(A. 1 6)} {\geq} \frac {1}{2 c _ {p} L} \frac {1}{\sqrt {\sum_ {s = 0} ^ {T - 1} f _ {i _ {s}} \left(\mathbf {x} _ {s}\right) - f ^ {\star}}} \geq \frac {1}{\left(2 c _ {p} L \left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\|\right) ^ {2}}. \tag {B.30}
$$
Plugging in the lower bound of  $\eta_{T - 1}$  to (B.26), we obtain:
$$
\left| \left| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \leq \left| \left| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right| \right| ^ {2} - \frac {1}{\left(2 c _ {p} L \left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\lvert\right) ^ {2}} \left\langle \nabla f _ {i _ {t}} (\mathbf {x} _ {t}), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\rangle . \tag {B.31}
$$
Plugging in  $c_p = \frac{c_p^{\mathrm{scale}}}{\sqrt{f_{i_0}(\mathbf{x}_0) - f^*}}$ , we get
$$
\left| \left| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \leq \left| \left| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right| \right| ^ {2} - \frac {f _ {i _ {0}} \left(\mathbf {x} _ {0}\right) - f ^ {\star}}{\left(2 c _ {p} ^ {\text {s c a l e}} L \| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \|\right) ^ {2}} \langle \nabla f _ {i _ {t}} (\mathbf {x} _ {t}), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \rangle . \tag {B.32}
$$
case 1:  $f$  is convex.
For any  $t \geq 1$ , we take expectation conditional on  $i_0$  on both sides and get:
$$
\mathbb {E} [ \| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \| ^ {2} | i _ {0} ] \leq \mathbb {E} [ \| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \| ^ {2} | i _ {0} ] - \frac {f _ {i _ {0}} (\mathbf {x} _ {0}) - f ^ {\star}}{\left(2 c _ {p} ^ {\text {s c a l e}} L \| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \|\right) ^ {2}} \mathbb {E} [ \langle \nabla f _ {i _ {t}} (\mathbf {x} _ {t}), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \rangle | i _ {0} ] \tag {B.33}
$$
$$
\stackrel {(A, 1)} {\leq} \mathbb {E} [ | | \mathbf {x} _ {t} - \mathbf {x} ^ {\star} | | ^ {2} | i _ {0} ] - \frac {f _ {i _ {0}} (\mathbf {x} _ {0}) - f ^ {\star}}{(2 c _ {p} ^ {\mathrm {s c a l e}} L | | \mathbf {x} _ {0} - \mathbf {x} ^ {\star} | |) ^ {2}} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} | i _ {0} ].
$$
Summing up from  $t = 1$  to  $t = T$  and dividing by  $T$ , we obtain:
$$
\frac {1}{T} \sum_ {t = 1} ^ {T} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} | i _ {0} ] \leq 4 L \left(c _ {p} ^ {\text {s c a l e}}\right) ^ {2} \frac {\left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\| ^ {2}}{f _ {i _ {0}} (\mathbf {x} _ {0}) - f ^ {\star}} \frac {L \mathbb {E} \left[ \left\| \mathbf {x} _ {1} - \mathbf {x} ^ {\star} \right\| ^ {2} | i _ {0} \right]}{T}. \tag {B.34}
$$
Note that  $\mathbb{E}[||\mathbf{x}_1 - \mathbf{x}^\star ||^2 |i_0] = ||\mathbf{x}_1 - \mathbf{x}^\star ||^2\leq ||\mathbf{x}_0 - \mathbf{x}^\star ||^2$  due to (B.31). We thus get:
$$
\frac {1}{T} \sum_ {t = 1} ^ {T} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} | i _ {0} ] \leq \left(4 L \left(c _ {p} ^ {\text {s c a l e}}\right) ^ {2} \frac {\left| \left| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right| \right| ^ {2}}{f _ {i _ {0}} (\mathbf {x} _ {0}) - f ^ {\star}}\right) \frac {L \left| \left| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right| \right| ^ {2}}{T} \tag {B.35}
$$
Taking expectation w.r.t  $i_0$  on both sides, we arrive at:
$$
\frac {1}{T} \sum_ {t = 1} ^ {T} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ] \leq A \frac {L \left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\| ^ {2}}{T} \text {w i t h} A := 4 L \left(c _ {p} ^ {\text {s c a l e}}\right) ^ {2} \mathbb {E} _ {i _ {0}} \left[ \frac {\left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\| ^ {2}}{f _ {i _ {0}} (\mathbf {x} _ {0}) - f ^ {\star}} \right]. \tag {B.36}
$$
case 2:  $f$  is strongly-convex
For any  $t \geq 1$ , we take expectation conditional on  $i_0$  on both sides of (B.32) and get:
$$
\begin{array}{l} \mathbb {E} [ | | \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} | | ^ {2} | i _ {0} ] \leq \mathbb {E} [ | | \mathbf {x} _ {t} - \mathbf {x} ^ {\star} | | ^ {2} | i _ {0} ] - \frac {f _ {i _ {0}} (\mathbf {x} _ {0}) - f ^ {\star}}{(2 c _ {p} ^ {\mathrm {s c a l e}} L | | \mathbf {x} _ {0} - \mathbf {x} ^ {\star} | |) ^ {2}} \mathbb {E} [ \langle \nabla f _ {i _ {t}} (\mathbf {x} _ {t}), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \rangle | i _ {0} ] \\ \stackrel {(A. 2)} {\leq} \mathbb {E} \left[ \left| \left| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \mid i _ {0} \right] - \frac {f _ {i _ {0}} \left(\mathbf {x} _ {0}\right) - f ^ {\star}}{\left(2 c _ {p} ^ {\text {s c a l e}} L \left| \left| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right| \right|\right) ^ {2}} \mathbb {E} \left[ f \left(\mathbf {x} _ {t}\right) - f ^ {\star} + \frac {\mu}{2} \left| \left| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \mid i _ {0} \right]. \tag {B.37} \\ \end{array}
$$
Note that  $f(\mathbf{x}_t) - f^{\star}\geq \frac{\mu}{2} ||\mathbf{x}_t - \mathbf{x}^{\star}||^2$  due to strong convexity, we obtain:
$$
\mathbb {E} \left[ \left| \left| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \mid i _ {0} \right] \leq \left(1 - \frac {\left(f _ {i _ {0}} \left(\mathbf {x} _ {0}\right) - f ^ {\star}\right) \mu}{\left(2 c _ {p} ^ {\text {s c a l e}} L \left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\|\right) ^ {2}}\right) \mathbb {E} \left[ \left| \left| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \mid i _ {0} \right]. \tag {B.38}
$$
For any  $T \geq 1$ , unrolling gives:
$$
\begin{array}{l} \mathbb {E} \left[ \left| \left| \mathbf {x} _ {T + 1} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \mid i _ {0} \right] \leq \left(1 - \frac {\left(f _ {i _ {0}} (\mathbf {x} _ {0}) - f ^ {\star}\right) \mu}{\left(2 c _ {p} ^ {\text {s c a l e}} L \left| \left| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right| \right|\right) ^ {2}}\right) ^ {T} \mathbb {E} \left[ \left| \left| \mathbf {x} _ {1} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \mid i _ {0} \right] \tag {B.39} \\ \leq \left(1 - \frac {\left(f _ {i _ {0}} \left(\mathbf {x} _ {0}\right) - f ^ {\star}\right) \mu}{\left(2 c _ {p} ^ {\text {s c a l e}} L \left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\vert\right) ^ {2}}\right) ^ {T} \left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\| ^ {2}. \\ \end{array}
$$
The claim of Theorem 6 follows by taking expectation w.r.t  $i_0$  on both sides.
AdaSLS: We only highlight the difference from AdaSPS. Plugging in the upper bound of  $\eta_t$  from Lemma 16 and using the line-search condition, we get:
$$
\begin{array}{l} \left| \left| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right| \right| ^ {2} \stackrel {(A. 1 7)} {\leq} \left| \left| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right| \right| ^ {2} - 2 \eta_ {t} \left\langle \nabla f _ {i _ {t}} \left(\mathbf {x} _ {t}\right), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\rangle + \eta_ {t} \frac {\gamma_ {t} \left| \left| \nabla f _ {i _ {t}} \left(\mathbf {x} _ {t}\right) \right| \right| ^ {2}}{c _ {l} \sqrt {\sum_ {s = 0} ^ {t} \gamma_ {s} \left| \left| \nabla f _ {i _ {s}} \left(\mathbf {x} _ {s}\right) \right| \right| ^ {2}}} \tag {B.40} \\ \leq | | \mathbf {x} _ {t} - \mathbf {x} ^ {\star} | | ^ {2} - 2 \eta_ {t} \langle \nabla f _ {i _ {t}} (\mathbf {x} _ {t}), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \rangle + \eta_ {t} \frac {f _ {i _ {t}} (\mathbf {x} _ {t}) - f ^ {\star}}{c _ {l} \rho \sqrt {\sum_ {s = 0} ^ {t} \gamma_ {s} | | \nabla f _ {i _ {s}} (\mathbf {x} _ {s}) | | ^ {2}}}. \\ \end{array}
$$
Since  $c_{l}\rho \sqrt{\sum_{s = 0}^{t}\gamma_{s}||\nabla f_{i_{s}}(\mathbf{x}_{s})||^{2}}\geq 1$ , we obtain the same equation as (B.26). To find a lower bound of  $\eta_{T - 1}$  we rearrange (B.25) as:
$$
\eta_ {t} \left(f _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - f ^ {\star}\right) \leq \left\| \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\| ^ {2} - \left\| \mathbf {x} _ {t + 1} - \mathbf {x} ^ {\star} \right\| ^ {2}, \tag {B.41}
$$
the left-hand-side of which can be lower bounded by:
$$
\eta_ {t} \left(f _ {i _ {t}} (\mathbf {x} _ {t}) - f ^ {\star}\right) \geq \eta_ {t} \rho \gamma_ {t} \| \nabla f _ {i _ {t}} (\mathbf {x} _ {t}) \| ^ {2} \stackrel {(A. 1 7)} {\geq} \min  \left\{\frac {1 - \rho}{L}, \gamma_ {\max } \right\} \frac {\rho \gamma_ {t} \| \nabla f _ {i _ {t}} (\mathbf {x} _ {t}) \| ^ {2}}{c _ {l} \sqrt {\sum_ {s = 0} ^ {t} \gamma_ {s} \| \nabla f _ {i _ {s}} (\mathbf {x} _ {s}) \| ^ {2}}} \cdot \tag {B.42}
$$
Summing over  $t = 0$  to  $t = T - 1$  gives:
$$
\min  \left\{\frac {1 - \rho}{L}, \gamma_ {\max } \right\} \frac {\rho}{c _ {l}} \sqrt {\sum_ {s = 0} ^ {T - 1} \gamma_ {s} \left\| \nabla f _ {i _ {s}} (\mathbf {x} _ {s}) \right\| ^ {2}} \stackrel {(A. 6)} {\leq} \min  \left\{\frac {1 - \rho}{L}, \gamma_ {\max } \right\} \frac {\rho}{c _ {l}} \sum_ {t = 0} ^ {T - 1} \frac {\gamma_ {t} \left\| \nabla f _ {i _ {t}} (\mathbf {x} _ {t}) \right\| ^ {2}}{\sqrt {\sum_ {s = 0} ^ {t} \gamma_ {s} \left\| \nabla f _ {i _ {s}} (\mathbf {x} _ {s}) \right\| ^ {2}}} \leq \| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \| ^ {2}. \tag {B.43}
$$
This implies that:
$$
\eta_ {T - 1} \stackrel {(A. 1 7)} {\geq} \frac {\min  \left\{\frac {1 - \rho}{L} , \gamma_ {\max } \right\}}{c _ {l} \sqrt {\sum_ {s = 0} ^ {T - 1} \gamma_ {s} \left\| \nabla f _ {i _ {s}} (\mathbf {x} _ {s}) \right\| ^ {2}}} \stackrel {(B. 4 3)} {\geq} \frac {\rho \min  ^ {2} \left\{\frac {1 - \rho}{L} , \gamma_ {\max } \right\}}{c _ {l} ^ {2} \left\| \mathbf {x} _ {0} - \mathbf {x} ^ {\star} \right\| ^ {2}}. \tag {B.44}
$$
After plugging the above into (B.26), the remaining proof follows from the same routine as shown for AdaSPS.
Proofs for Loopless Variance-Reduction
B.4 Statement and Proof of Lemma 18
The following Lemma provides us with the guarantee that as  $\mathbf{w}_t, \mathbf{x}_t \to \mathbf{x}^\star$ ,  $\mathbb{E}_{i_t}[F_{i_t}(\mathbf{x}_t) - F_{i_t}^\star] \to 0$ , which implies diminishing variance.
Lemma 18. Assume each  $f_{i}$  is convex and  $L$ -smooth, for any  $t \geq 0$ , the iterates generated by Algorithm 1 satisfy:
$$
\mathbb {E} _ {i _ {t}} \left[ F _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - F _ {i _ {t}} ^ {\star} \right] \leq f (\mathbf {x} _ {t}) - f ^ {\star} + \frac {1}{2 \mu_ {F}} \mathbb {E} _ {i _ {t}} \left[ \left| | \nabla f _ {i _ {t}} \left(\mathbf {w} _ {t}\right) - \nabla f _ {i _ {t}} \left(\mathbf {x} ^ {\star}\right) \right| \right. ^ {2} ]. \tag {B.45}
$$
Proof. By definition of  $F_{i_t}(\mathbf{x})$ , we have:
$$
\begin{array}{l} F _ {i _ {t}} (\mathbf {x} _ {t}) - F _ {i _ {t}} ^ {\star} \\ = F _ {i t} \left(\mathbf {x} _ {t}\right) - F _ {i t} \left(\mathbf {x} ^ {\star}\right) + F _ {i t} \left(\mathbf {x} ^ {\star}\right) - F _ {i t} ^ {\star} \tag {B.46} \\ = f _ {i _ {t}} (\mathbf {x} _ {t}) - f _ {i _ {t}} (\mathbf {x} ^ {\star}) + \left(\mathbf {x} _ {t} - \mathbf {x} ^ {\star}\right) ^ {T} (\nabla f (\mathbf {w} _ {t}) - \nabla f _ {i _ {t}} (\mathbf {w} _ {t})) - \frac {\mu_ {F}}{2} | | \mathbf {x} _ {t} - \mathbf {x} ^ {\star} | | ^ {2} + F _ {i _ {t}} (\mathbf {x} ^ {\star}) - F _ {i _ {t}} ^ {\star}. \\ \end{array}
$$
By  $\mu_F$ -strong convexity of  $F_{it}(\mathbf{x})$ , we obtain:
$$
\begin{array}{l} F _ {i _ {t}} \left(\mathbf {x} ^ {\star}\right) - F _ {i _ {t}} ^ {\star} \leq \frac {1}{2 \mu_ {F}} \left\| \nabla F _ {i _ {t}} \left(\mathbf {x} ^ {\star}\right) \right\| ^ {2} \tag {B.47} \\ = \frac {1}{2 \mu_ {F}} | | \nabla f _ {i _ {t}} (\mathbf {x} ^ {\star}) - \nabla f _ {i _ {t}} (\mathbf {w} _ {t}) + \nabla f (\mathbf {w} _ {t}) + \mu_ {F} (\mathbf {x} ^ {\star} - \mathbf {x} _ {t}) | | ^ {2}. \\ \end{array}
$$
Plugging (B.47) into (B.46), taking expectation w.r.t the randomness  $i_t$  on both sides gives:
$$
\begin{array}{l} \mathbb {E} _ {i _ {t}} \left[ F _ {i _ {t}} (\mathbf {x} _ {t}) - F _ {i _ {t}} ^ {\star} \right] \\ \leq f (\mathbf {x} _ {t}) - f (\mathbf {x} ^ {\star}) - \frac {\mu_ {F}}{2} | | \mathbf {x} _ {t} - \mathbf {x} ^ {\star} | | ^ {2} + \mathbb {E} _ {i _ {t}} [ \frac {1}{2 \mu_ {F}} | | \nabla f _ {i _ {t}} (\mathbf {x} ^ {\star}) - \nabla f _ {i _ {t}} (\mathbf {w} _ {t}) + \nabla f (\mathbf {w} _ {t}) + \mu_ {F} (\mathbf {x} ^ {\star} - \mathbf {x} _ {t}) | | ^ {2} ] \\ = f (\mathbf {x} _ {t}) - f (\mathbf {x} ^ {\star}) - \frac {\mu_ {F}}{2} | | \mathbf {x} _ {t} - \mathbf {x} ^ {\star} | | ^ {2} + \frac {\mu_ {F}}{2} | | \mathbf {x} _ {t} - \mathbf {x} ^ {\star} | | ^ {2} + \frac {1}{2 \mu_ {F}} \mathbb {E} _ {i _ {t}} [ | | \nabla f _ {i _ {t}} (\mathbf {x} ^ {\star}) - \nabla f _ {i _ {t}} (\mathbf {w} _ {t}) + \nabla f (\mathbf {w} _ {t}) | | ^ {2} ] \\ + \frac {1}{\mu_ {F}} \mathbb {E} _ {i _ {t}} \left[ \left\langle \nabla f _ {i _ {t}} \left(\mathbf {x} ^ {\star}\right) - \nabla f _ {i _ {t}} \left(\mathbf {w} _ {t}\right) + \nabla f \left(\mathbf {w} _ {t}\right), \mu_ {F} \left(\mathbf {x} ^ {\star} - \mathbf {x} _ {t}\right) \right\rangle \right] \\ = f \left(\mathbf {x} _ {t}\right) - f \left(\mathbf {x} ^ {\star}\right) + \frac {1}{2 \mu_ {F}} \mathbb {E} _ {i _ {t}} \left[ \left\| \nabla f _ {i _ {t}} \left(\mathbf {x} ^ {\star}\right) - \nabla f _ {i _ {t}} \left(\mathbf {w} _ {t}\right) + \nabla f (\mathbf {w} _ {t}) \right\| ^ {2} \right] \\ \leq f \left(\mathbf {x} _ {t}\right) - f \left(\mathbf {x} ^ {\star}\right) + \frac {1}{2 \mu_ {F}} \mathbb {E} _ {i _ {t}} \left[ \left| \left| \nabla f _ {i _ {t}} \left(\mathbf {x} ^ {\star}\right) - \nabla f _ {i _ {t}} \left(\mathbf {w} _ {t}\right) \right| \right| ^ {2} \right]. \tag {B.48} \\ \end{array}
$$
B.5 Proof of Theorem 7
Proof. Recall the proof technique that gives equation (B.6) and (B.14) in Theorem 1. Following the same routine, we arrive at:
$$
\sum_ {t = 0} ^ {T - 1} \left\langle \nabla F _ {i _ {t}} \left(\mathbf {x} _ {t}\right), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \right\rangle \leq \tau \sqrt {\sum_ {t = 0} ^ {T - 1} F _ {i _ {t}} \left(\mathbf {x} _ {t}\right) - F _ {i _ {t}} ^ {\star}}. \tag {B.49}
$$
where  $\tau = (2c_p(L + \mu_F)D^2 +\frac{1}{c_p})$  for AdaSVRPS and  $\tau = \max \left\{\frac{L + \mu_F}{(1 - \rho)\sqrt{\rho}},\frac{1}{\gamma_{\mathrm{max}}\sqrt{\rho}}\right\} c_lD^2 +\frac{1}{c_l\sqrt{\rho}}$  for AdaSVRLS. The difference is due to the fact that  $F_{i_t}(\mathbf{x})$  is  $(L + \mu_F)$ -smooth. Taking the expectation, using the fact that  $\mathbb{E}[\nabla F_{i_t}(\mathbf{x}_t)] = \mathbb{E}[\nabla f_{i_t}(\mathbf{x}_t) + \nabla f(\mathbf{w}_t) - \nabla f_{i_t}(\mathbf{w}_t)] = \nabla f(\mathbf{x}_t)$  and applying Lemma 18, we end up with:
$$
\sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ] \stackrel {(B. 4 8)} {\leq} \tau \sqrt {\sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ] + \frac {1}{2 \mu_ {F}} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ | | \nabla f _ {i _ {t}} (\mathbf {w} _ {t}) - \nabla f _ {i _ {t}} (\mathbf {x} ^ {\star}) | | ^ {2} ]}. \tag {B.50}
$$
Taking the square gives:
$$
\left(\sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ]\right) ^ {2} \leq \tau^ {2} \left(\sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ] + \frac {1}{2 \mu_ {F}} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ | | \nabla f _ {i _ {t}} (\mathbf {w} _ {t}) - \nabla f _ {i _ {t}} (\mathbf {x} ^ {\star}) | | ^ {2} ]\right). \tag {B.51}
$$
Define the Lyapunov function:  $\mathcal{Z}_{t + 1} = \frac{1}{2(1 - a)}\frac{\tau^2}{p_{t + 1}\mu_F} ||\nabla f_{i_{t + 1}}(\mathbf{w}_{t + 1}) - \nabla f_{i_{t + 1}}(\mathbf{x}^\star)||^2$ . It follows that:
$$
\begin{array}{l} \mathbb {E} [ \mathcal {Z} _ {t + 1} ] = \frac {1}{2 (1 - a)} \frac {\tau^ {2}}{p _ {t + 1} \mu_ {F}} \mathbb {E} [ | | \nabla f _ {i _ {t + 1}} (\mathbf {w} _ {t + 1}) - \nabla f _ {i _ {t + 1}} (\mathbf {x} ^ {\star}) | | ^ {2} ] \\ = \frac {\tau^ {2}}{2 (1 - a) \mu_ {F}} \mathbb {E} [ | | \nabla f _ {i _ {t + 1}} (\mathbf {x} _ {t}) - \nabla f _ {i _ {t + 1}} (\mathbf {x} ^ {\star}) | | ^ {2} ] + \frac {1 - p _ {t + 1}}{2 (1 - a)} \frac {\tau^ {2}}{p _ {t + 1} \mu_ {F}} \mathbb {E} [ | | \nabla f _ {i _ {t}} (\mathbf {w} _ {t}) - \nabla f _ {i _ {t}} (\mathbf {x} ^ {\star}) | | ^ {2} ] \\ \stackrel {(A. 5)} {\leq} \frac {\tau^ {2}}{2 (1 - a) \mu_ {F}} \mathbb {E} [ 2 L (f _ {i _ {t + 1}} (\mathbf {x} _ {t}) - f _ {i _ {t + 1}} (\mathbf {x} ^ {\star}) - \langle \nabla f _ {i _ {t + 1}} (\mathbf {x} ^ {\star}), \mathbf {x} _ {t} - \mathbf {x} ^ {\star} \rangle) ] + \frac {(1 - p _ {t + 1}) p _ {t}}{p _ {t + 1}} \mathbb {E} [ \mathcal {Z} _ {t} ] \\ = \frac {L}{(1 - a) \mu_ {F}} \tau^ {2} \left(\mathbb {E} \left[ f \left(\mathbf {x} _ {t}\right) - f ^ {\star} \right]\right) + \frac {(1 - p _ {t + 1}) p _ {t}}{p _ {t + 1}} \mathbb {E} \left[ \mathcal {Z} _ {t} \right]. \tag {B.52} \\ \end{array}
$$
Adding  $\sum_{t=0}^{T-1} \mathbb{E}\left[\mathcal{Z}_{t+1}\right]$  to both sides of (B.51) and substituting the above upper bound, we get:
$$
\begin{array}{l} \left(\sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ]\right) ^ {2} + \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ \mathcal {Z} _ {t + 1} ] \leq (1 + \frac {L}{(1 - a) \mu_ {F}}) \tau^ {2} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ] \tag {B.53} \\ + \sum_ {t = 0} ^ {T - 1} \left((1 - a) p _ {t} + \frac {(1 - p _ {t + 1}) p _ {t}}{p _ {t + 1}}\right) \mathbb {E} [ \mathcal {Z} _ {t} ]. \\ \end{array}
$$
Rearranging and dropping  $\mathbb{E}[\mathcal{Z}_T] \geq 0$  gives:
$$
\begin{array}{l} \left(\sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ]\right) ^ {2} \leq (1 + \frac {L}{(1 - a) \mu_ {F}}) \tau^ {2} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ] + \sum_ {t = 1} ^ {T - 1} \left((1 - a) p _ {t} + \frac {(1 - p _ {t + 1}) p _ {t}}{p _ {t + 1}} - 1\right) \mathbb {E} [ \mathcal {Z} _ {t} ] \\ + \left((1 - a) p _ {0} + \frac {\left(1 - p _ {1}\right) p _ {0}}{p _ {1}}\right) \mathbb {E} \left[ \mathcal {Z} _ {0} \right]. \tag {B.54} \\ \end{array}
$$
By our choice of  $p_t$ , we have:
$$
(1 - a) p _ {t} + \frac {(1 - p _ {t + 1}) p _ {t}}{p _ {t + 1}} - 1 = \frac {p _ {t}}{p _ {t + 1}} - a p _ {t} - 1 = \frac {a t + a + 1}{a t + 1} - \frac {a}{(a t + 1)} - 1 = \frac {0}{2 (a t + 1)} = 0. \tag {B.55}
$$
Therefore, it holds that:
$$
\left(\sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ]\right) ^ {2} \leq \left(1 + \frac {L}{(1 - a) \mu_ {F}}\right) \tau^ {2} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ] + \mathbb {E} [ \mathcal {Z} _ {0} ]. \tag {B.56}
$$
Further, by  $L$ -smoothness and convexity of  $f$ , we have
$$
\mathbb {E} \left[ \mathcal {Z} _ {0} \right] = \frac {1}{2 (1 - a)} \frac {\tau^ {2}}{p _ {0} \mu_ {F}} \mathbb {E} \left[ \left| \left| \nabla f _ {i _ {0}} (\mathbf {x} _ {0}) - \nabla f _ {i _ {0}} (\mathbf {x} ^ {\star}) \right| \right| ^ {2} \right] \stackrel {(A. 5)} {\leq} \frac {L \tau^ {2}}{(1 - a) \mu_ {F}} \left(f (\mathbf {x} _ {0}) - f ^ {\star}\right). \tag {B.57}
$$
Hence, we obtain:
$$
\left(\sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ]\right) ^ {2} \leq (1 + \frac {2 L}{(1 - a) \mu_ {F}}) \tau^ {2} \sum_ {t = 0} ^ {T - 1} \mathbb {E} [ f (\mathbf {x} _ {t}) - f ^ {\star} ]. \tag {B.58}
$$
It follows that: