SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\left\{ \begin{array}{l} B _ {1} = \frac {8 L ^ {2}}{3} + \frac {1 4 N L ^ {2}}{M}, \\ B _ {2} = \frac {2}{M} + 1 + \frac {5}{6 L ^ {2}} + \frac {8 N}{3 M L ^ {2}}. \end{array} \right. \tag {25}
$$
Proof. We start with Assumption 5. Using the inequality  $\frac{1}{2} \| a \|^2 - \| b \|^2 \leq \| a - b \|^2$ , we have for  $t = 1, \ldots, T$  and  $i = 1, \ldots, n$ :
$$
\begin{array}{l} \frac {1}{2} \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} - \| \nabla f (w _ {*}; \pi^ {(t)} (i)) \| ^ {2} \\ \leq \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) - \nabla f (w _ {*}; \pi^ {(t)} (i)) \| ^ {2} \\ \stackrel {(1 3)} {\leq} M \langle \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) - \nabla f (w _ {*}; \pi^ {(t)} (i)), w _ {i - 1} ^ {(t)} - w _ {*} \rangle + N \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2} \\ = M \langle \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)), w _ {i - 1} ^ {(t)} - w _ {*} \rangle - M \langle \nabla f (w _ {*}; \pi^ {(t)} (i)), w _ {i - 1} ^ {(t)} - w _ {*} \rangle \\ + N \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2}, \\ \end{array}
$$
This statement is equivalent to
$$
\begin{array}{l} - \langle \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)), w _ {i - 1} ^ {(t)} - w _ {*} \rangle \leq - \frac {1}{2 M} \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} + \frac {1}{M} \| \nabla f (w _ {*}; \pi^ {(t)} (i)) \| ^ {2} \\ - \left\langle \nabla f (w _ {*}; \pi^ {(t)} (i)), w _ {i - 1} ^ {(t)} - w _ {*} \right\rangle \\ + \frac {N}{M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2}, \tag {26} \\ \end{array}
$$
For any  $w_{*}\in W^{*}$  , from the update (19) we have,
$$
\| w _ {i} ^ {(t)} - w _ {*} \| ^ {2} \stackrel {(1 9)} {=} \| w _ {i - 1} ^ {(t)} - w _ {*} \| ^ {2} - \frac {2 \eta_ {t}}{n} \langle \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)), w _ {i - 1} ^ {(t)} - w _ {*} \rangle + \frac {\eta_ {t} ^ {2}}{n ^ {2}} \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2}
$$
$$
\begin{array}{l} \stackrel {(2 6)} {\leq} \| w _ {i - 1} ^ {(t)} - w _ {*} \| ^ {2} - \frac {2 \eta_ {t}}{2 M n} \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} + \frac {2 \eta_ {t}}{M n} \| \nabla f (w _ {*}; \pi^ {(t)} (i)) \| ^ {2} \\ - \frac {2 \eta_ {t}}{n} \langle \nabla f (w _ {*}; \pi^ {(t)} (i)), w _ {i - 1} ^ {(t)} - w _ {*} \rangle + \frac {2 \eta_ {t} N}{M n} \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2} \\ + \frac {\eta_ {t} ^ {2}}{n ^ {2}} \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(a)} {\leq} \| w _ {i - 1} ^ {(t)} - w _ {*} \| ^ {2} - \frac {\eta_ {t}}{2 M n} \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} + \frac {2 \eta_ {t}}{M n} \| \nabla f (w _ {*}; \pi^ {(t)} (i)) \| ^ {2} \\ - \frac {2 \eta_ {t}}{n} \langle \nabla f (w _ {*}; \pi^ {(t)} (i)), w _ {i - 1} ^ {(t)} - w _ {*} \rangle + \frac {2 \eta_ {t} N}{M n} \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2} \\ = \| w _ {i - 1} ^ {(t)} - w _ {*} \| ^ {2} - \frac {\eta_ {t}}{2 M n} \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} + \frac {2 \eta_ {t}}{M n} \| \nabla f (w _ {*}; \pi^ {(t)} (i)) \| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} - \frac {2 \eta_ {t}}{n} \langle \nabla f (w _ {*}; \pi^ {(t)} (i)), w _ {i - 1} ^ {(t)} - w _ {0} ^ {(t)} \rangle - \frac {2 \eta_ {t}}{n} \langle \nabla f (w _ {*}; \pi^ {(t)} (i)), w _ {0} ^ {(t)} - w _ {*} \rangle \\ + \frac {2 \eta_ {t} N}{M n} \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2} \\ \stackrel {(b)} {\leq} \left\| w _ {i - 1} ^ {(t)} - w _ {*} \right\| ^ {2} - \frac {\eta_ {t}}{2 M n} \left\| \nabla f \left(w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)\right) \right\| ^ {2} + \frac {2 \eta_ {t}}{M n} \left\| \nabla f \left(w _ {*}; \pi^ {(t)} (i)\right) \right\| ^ {2} \\ + \frac {\eta_ {t}}{n} \| \nabla f (w _ {*}; \pi^ {(t)} (i)) \| ^ {2} + \frac {\eta_ {t}}{n} \| w _ {i - 1} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2} \\ - \frac {2 \eta_ {t}}{n} \langle \nabla f (w _ {*}; \pi^ {(t)} (i)), w _ {0} ^ {(t)} - w _ {*} \rangle + \frac {2 \eta_ {t} N}{M n} \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2}, \\ \end{array}
$$
where  $(a)$  follows since  $\eta_t \leq \frac{n}{2M}$  and  $(b)$  follows by the inequality  $2\langle a, b \rangle \leq \|a\|^2 \|b\|^2$ .
Note that  $\frac{1}{n}\sum_{i=1}^{n}\langle\nabla f(w_ {*};\pi^{(t)}(i)),w_0^{(t)}-w_*\rangle=\langle\nabla F(w_ {*}),w_0^{(t)}-w_*\rangle=0$  since  $w_*$  is a global solution of  $F$ . Now we sum the derived statement for  $i=1,\ldots,n$  and get
$$
\begin{array}{l} \left\| w _ {n} ^ {(t)} - w _ {*} \right\| ^ {2} \leq \left\| w _ {0} ^ {(t)} - w _ {*} \right\| ^ {2} - \frac {\eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| \nabla f \left(w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)\right) \right\| ^ {2} \\ + \eta_ {t} \left(\frac {2}{M} + 1\right) \frac {1}{n} \sum_ {i = 1} ^ {n} \| \nabla f (w _ {*}; \pi^ {(t)} (i)) \| ^ {2} + \frac {\eta_ {t}}{n} \sum_ {i = 1} ^ {n} \| w _ {i - 1} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2} \\ + \frac {2 N \eta_ {t}}{M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(8), (2 2)} {\leq} \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} - \frac {\eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| \nabla f \left(w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)\right) \| ^ {2} \\ + \left(\frac {2}{M} + 1\right) \eta_ {t} \sigma_ {*} ^ {2} + \frac {8 L ^ {2} \eta_ {t} ^ {3}}{3} \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} + \frac {1 6 L ^ {2} \eta_ {t} ^ {5}}{3} \sigma_ {*} ^ {2} + 2 \eta_ {t} ^ {3} \sigma_ {*} ^ {2} \\ + \frac {2 N \eta_ {t}}{M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(2 3)} {\leq} \left\| w _ {0} ^ {(t)} - w _ {*} \right\| ^ {2} - \frac {\eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| \nabla f \left(w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)\right) \right\| ^ {2} \\ + \left(\frac {2}{M} + 1\right) \eta_ {t} \sigma_ {*} ^ {2} + \frac {8 L ^ {2} \eta_ {t} ^ {3}}{3} \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} + \frac {1 6 L ^ {2} \eta_ {t} ^ {5}}{3} \sigma_ {*} ^ {2} + 2 \eta_ {t} ^ {3} \sigma_ {*} ^ {2} \\ + \frac {1 6 N L ^ {2} \eta_ {t} ^ {3}}{3 M} \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} + \frac {3 2 N L ^ {2} \eta_ {t} ^ {5}}{3 M} \sigma_ {*} ^ {2} + \frac {4 N \eta_ {t} ^ {3}}{M} \sigma_ {*} ^ {2} \\ + \frac {8 N L ^ {2} \eta_ {t} ^ {3}}{M n} \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} + \frac {1 6 N L ^ {2} \eta_ {t} ^ {5}}{M n} \sigma_ {*} ^ {2}, \\ \end{array}
$$
where we apply the derivations from Lemma 3. Now noting that  $\eta_t \leq \frac{1}{2L}$ ,  $n \leq 1$  and rearranging the terms we get:
$$
\begin{array}{l} \left\| w _ {n} ^ {(t)} - w _ {*} \right\| ^ {2} \leq \left\| w _ {0} ^ {(t)} - w _ {*} \right\| ^ {2} - \frac {\eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \left\| \nabla f \left(w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)\right) \right\| ^ {2} \\ + \left(\frac {8 L ^ {2}}{3} + \frac {1 6 N L ^ {2}}{3 M} + \frac {8 N L ^ {2}}{M}\right) \eta_ {t} ^ {3} \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} \\ + \left(\frac {2}{M} + 1 + \frac {1}{3 L ^ {2}} + \frac {1}{2 L ^ {2}} + \frac {2 N}{3 M L ^ {2}} + \frac {N}{M L ^ {2}} + \frac {N}{M L ^ {2}}\right) \eta_ {t} \sigma_ {*} ^ {2} \\ \end{array}
$$
Since  $w_{n}^{(t)} = w_{0}^{(t + 1)} = \tilde{w}_{t}$ , we have the desired result in (24).
Proof. From (24) where  $B_{1}$  and  $B_{2}$  are defined in (25), we have
$$
\begin{array}{l} \left\| w _ {0} ^ {(t + 1)} - w _ {*} \right\| ^ {2} \\ \leq \left(1 + B _ {1} \eta_ {t} ^ {3}\right) \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} - \frac {\eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} + B _ {2} \eta_ {t} \sigma_ {*} ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(a)} {\leq} \left(1 + B _ {1} \eta_ {t} ^ {3}\right) \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} - \frac {\gamma}{\gamma + 1} \frac {\eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| \nabla f (w _ {0} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} \\ + \frac {\eta_ {t} \gamma}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| \nabla f (w _ {i - 1} ^ {(t)}; \pi^ {(t)} (i)) - \nabla f (w _ {0} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} + B _ {2} \eta_ {t} \sigma_ {*} ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(2)} {\leq} \left(1 + B _ {1} \eta_ {t} ^ {3}\right) \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} - \frac {\gamma}{\gamma + 1} \frac {\eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| \nabla f (w _ {0} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} \\ + \frac {\eta_ {t} \gamma L ^ {2}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| w _ {i - 1} ^ {(t)} - w _ {0} ^ {(t)} \| ^ {2} + B _ {2} \sigma_ {*} ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(2 2)} {\leq} \left(1 + B _ {1} \eta_ {t} ^ {3}\right) \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} - \frac {\gamma}{\gamma + 1} \frac {\eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| \nabla f (w _ {0} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} \\ + \frac {\eta_ {t} ^ {3} \gamma L ^ {2}}{2 M} \left(\frac {8 L ^ {2}}{3} \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} + \frac {1 6 L ^ {2} \sigma_ {*} ^ {2}}{3} \cdot \eta_ {t} ^ {2} + 2 \sigma_ {*} ^ {2}\right) + B _ {2} \eta_ {t} \sigma_ {*} ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} = \left(1 + B _ {1} \eta_ {t} ^ {3} + \frac {4 \eta_ {t} ^ {3} \gamma L ^ {4}}{3 M}\right) \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} + \left(B _ {2} + \frac {\eta_ {t} ^ {2} \gamma L ^ {2}}{M} + \frac {8 \eta_ {t} ^ {4} \gamma L ^ {4}}{3 M}\right) \eta_ {t} \sigma_ {*} ^ {2} \\ - \frac {\gamma}{\gamma + 1} \frac {\eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \| \nabla f (w _ {0} ^ {(t)}; \pi^ {(t)} (i)) \| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(5)} {\leq} \left(1 + \eta_ {t} ^ {3} \left(B _ {1} + \frac {4 \gamma L ^ {4}}{3 M}\right)\right) \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} + \left(B _ {2} + \frac {\eta_ {t} ^ {2} \gamma L ^ {2}}{M} + \frac {8 \eta_ {t} ^ {4} \gamma L ^ {4}}{3 M}\right) \eta_ {t} \sigma_ {*} ^ {2} \\ - \frac {\gamma}{\gamma + 1} \frac {2 \mu \eta_ {t}}{2 M} \frac {1}{n} \sum_ {i = 1} ^ {n} \left[ f \left(w _ {0} ^ {(t)}; \pi^ {(t)} (i)\right) - f _ {i} ^ {*} \right] \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(3)} {\leq} \left(1 + \eta_ {t} ^ {3} \left(B _ {1} + \frac {4 \gamma L ^ {4}}{3 M}\right)\right) \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} + \left(B _ {2} + \frac {\eta_ {t} ^ {2} \gamma L ^ {2}}{M} + \frac {8 \eta_ {t} ^ {4} \gamma L ^ {4}}{3 M}\right) \eta_ {t} \sigma_ {*} ^ {2} \\ - \frac {\gamma}{\gamma + 1} \frac {\mu \eta_ {t}}{M} \left[ F \left(w _ {0} ^ {(t)}\right) - F _ {*} \right] \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(b)} {\leq} \left(1 + \eta_ {t} ^ {3} \left(B _ {1} + \frac {4 \gamma L ^ {4}}{3 M}\right)\right) \| w _ {0} ^ {(t)} - w _ {*} \| ^ {2} + \left(B _ {2} + \frac {\gamma}{4 M} + \frac {\gamma}{6 M}\right) \eta_ {t} \sigma_ {*} ^ {2} \\ - \frac {\gamma}{\gamma + 1} \frac {\mu \eta_ {t}}{M} [ F (w _ {0} ^ {(t)}) - F _ {*} ], \\ \end{array}
$$
where  $(a)$  follows since  $-\|b\|^2 \leq \gamma \|a - b\|^2 - \frac{\gamma}{\gamma+1} \|a\|^2$  for any  $\gamma > 0$  and  $(b)$  follows since  $\eta_t \leq \frac{1}{2L}$ . Since  $w_0^{(t+1)} = \tilde{w}_t$ , we obtain the desired result in (14).
Proof. For  $t = 1, \ldots, T = \frac{\lambda}{\varepsilon^{3/2}}$  for some  $\lambda > 0$
$$
\begin{array}{l} \eta_ {t} = \left(1 + C _ {1} D ^ {3} \varepsilon^ {3 / 2}\right) \eta_ {t - 1} = \left(1 + C _ {1} D ^ {3} \varepsilon^ {3 / 2}\right) ^ {t} \eta_ {0} \leq \left(1 + C _ {1} D ^ {3} \varepsilon^ {3 / 2}\right) ^ {T} \eta_ {0} \\ = (1 + C _ {1} D ^ {3} \varepsilon^ {3 / 2}) ^ {\lambda / \varepsilon^ {3 / 2}} \eta_ {0} = (1 + C _ {1} D ^ {3} \varepsilon^ {3 / 2}) ^ {\lambda / \varepsilon^ {3 / 2}} \frac {D \sqrt {\bar {\varepsilon}}}{(1 + C _ {1} D ^ {3} \varepsilon^ {3 / 2}) \exp (\lambda C _ {1} D ^ {3})} \\ \leq \frac {D \sqrt {\varepsilon}}{\left(1 + C _ {1} D ^ {3} \varepsilon^ {3 / 2}\right)} \leq \min  \left\{\frac {n}{2 M}, \frac {1}{2 L} \right\}, \tag {27} \\ \end{array}
$$
since  $(1 + x)^{1 / x}\leq e,x > 0$  .From (14),we have
$$
\left[ F \left(\tilde {w} _ {t - 1}\right) - F _ {*} \right] \leq \frac {1}{C _ {3}} \left(\frac {1}{\eta_ {t}} + C _ {1} \eta_ {t} ^ {2}\right) \| \tilde {w} _ {t - 1} - w _ {*} \| ^ {2} - \frac {1}{C _ {3} \eta_ {t}} \| \tilde {w} _ {t} - w _ {*} \| ^ {2} + \frac {C _ {2}}{C _ {3}} \sigma_ {*} ^ {2}. \tag {28}
$$
We proceed to prove the following inequality for  $t = 1,\ldots ,T$
$$
\frac {1}{\eta_ {t}} + C _ {1} \eta_ {t} ^ {2} \leq \frac {1}{\eta_ {t - 1}}. \tag {29}
$$
From (27), and  $\eta_t = K\eta_{t-1}$  where  $K = (1 + C_1D^3\varepsilon^{3/2})$ , we have
$$
\begin{array}{l} C _ {1} \eta_ {t} ^ {2} = C _ {1} K ^ {2} \eta_ {t - 1} ^ {2} = C _ {1} K ^ {2} \frac {\eta_ {t - 1} ^ {3}}{\eta_ {t - 1}} \\ \leq C _ {1} K ^ {2} \frac {D ^ {3} \varepsilon^ {3 / 2}}{K ^ {3} \eta_ {t - 1}} = \frac {C _ {1} D ^ {3} \varepsilon^ {3 / 2}}{K \eta_ {t - 1}} \quad \text {s i n c e} \eta_ {t - 1} \leq \frac {D \sqrt {\varepsilon}}{K} \\ = \frac {K - 1}{K} \frac {1}{\eta_ {t - 1}} = \frac {1}{\eta_ {t - 1}} - \frac {1}{K \eta_ {t - 1}} \quad \text {s i n c e} K = \left(1 + C _ {1} D ^ {3} \varepsilon^ {3 / 2}\right) \\ = \frac {1}{\eta_ {t - 1}} - \frac {1}{K \eta_ {t - 1}} = \frac {1}{\eta_ {t - 1}} - \frac {1}{\eta_ {t}}, \quad \text {s i n c e} \eta_ {t} = K \eta_ {t - 1}. \\ \end{array}
$$
for  $t = 1,\ldots ,T$  . Hence, from (28), we have
$$
\begin{array}{l} \left[ F \left(\tilde {w} _ {t - 1}\right) - F _ {*} \right] \leq \frac {1}{C _ {3}} \left(\frac {1}{\eta_ {t}} + C _ {1} \eta_ {t} ^ {2}\right) \| \tilde {w} _ {t - 1} - w _ {*} \| ^ {2} - \frac {1}{C _ {3} \eta_ {t}} \| \tilde {w} _ {t} - w _ {*} \| ^ {2} + \frac {C _ {2}}{C _ {3}} \sigma_ {*} ^ {2} \\ \leq \frac {1}{C _ {3} \eta_ {t - 1}} \| \tilde {w} _ {t - 1} - w _ {*} \| ^ {2} - \frac {1}{C _ {3} \eta_ {t}} \| \tilde {w} _ {t} - w _ {*} \| ^ {2} + \frac {C _ {2}}{C _ {3}} \sigma_ {*} ^ {2}. \\ \end{array}
$$
Averaging the statement above for  $t = 1,\ldots ,T$  , we have
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 1} ^ {T} [ F (\tilde {w} _ {t - 1}) - F _ {*} ] \leq \frac {1}{C _ {3} \eta_ {0} T} \| \tilde {w} _ {0} - w _ {*} \| ^ {2} + \frac {C _ {2}}{C _ {3}} \sigma_ {*} ^ {2} \\ \stackrel {(a)} {=} \frac {K \exp (\lambda C _ {1} D ^ {3})}{C _ {3} D \sqrt {\varepsilon}} \frac {\varepsilon^ {3 / 2}}{\lambda} \| \tilde {w} _ {0} - w _ {*} \| ^ {2} + \frac {C _ {2}}{C _ {3}} \sigma_ {*} ^ {2} \\ = \frac {K \exp (\lambda C _ {1} D ^ {3})}{C _ {3} D \lambda} \| \tilde {w} _ {0} - w _ {*} \| ^ {2} \cdot \varepsilon + \frac {C _ {2}}{C _ {3}} \sigma_ {*} ^ {2}, \\ \end{array}
$$
where  $(a)$  follows since  $\eta_0 = \frac{D\sqrt{\varepsilon}}{K\exp(\lambda C_1D^3)}$  and  $T = \frac{\lambda}{\varepsilon^{3 / 2}}$
Proof. Choose  $\gamma = \frac{1}{L^2}$ , we have
Note that  $K = 1 + C_1D^3\varepsilon^{3 / 2}$  and  $C_1D^3 = 1 / \lambda$ , we get that  $K = 1 + 1 / T\leq 2$ . We continue from the statement of Theorem 2 and the choice  $C_1D^3\lambda = 1$ :
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 1} ^ {T} [ F (\tilde {w} _ {t - 1}) - F _ {*} ] \leq \frac {K \exp (\lambda C _ {1} D ^ {3})}{C _ {3} D \lambda} \| \tilde {w} _ {0} - w _ {*} \| ^ {2} \cdot \varepsilon + \frac {C _ {2}}{C _ {3}} \sigma_ {*} ^ {2} \\ \leq \frac {2}{C _ {1} D ^ {3} \lambda} \cdot \frac {C _ {1} D ^ {2} \exp \left(\lambda C _ {1} D ^ {3}\right)}{C _ {3}} \cdot \| \tilde {w} _ {0} - w _ {*} \| ^ {2} \cdot \varepsilon + \frac {C _ {2}}{C _ {3}} \sigma_ {*} ^ {2} \quad \text {s i n c e} K \leq 2 \\ \leq \frac {2 C _ {1} D ^ {2} e}{C _ {3}} \| \tilde {w} _ {0} - w _ {*} \| ^ {2} \cdot \varepsilon + \frac {C _ {2}}{C _ {3}} \sigma_ {*} ^ {2} \quad \text {s i n c e} C _ {1} D ^ {3} \lambda = 1 \\ \end{array}
$$
$$
\begin{array}{l} \leq \frac {2 C _ {1} D ^ {2} e}{C _ {3}} \| \tilde {w} _ {0} - w _ {*} \| ^ {2} \cdot \varepsilon + \frac {C _ {2}}{C _ {3}} P \varepsilon \\ \leq \left(\frac {2 C _ {1} D ^ {2} e}{C _ {3}} \| \tilde {w} _ {0} - w _ {*} \| ^ {2} + \frac {C _ {2} P}{C _ {3}}\right) \varepsilon = G \varepsilon \\ \end{array}
$$
equation (9)
$$
G = \frac {2 C _ {1} D ^ {2} e}{C _ {3}} \| \tilde {w} _ {0} - w _ {*} \| ^ {2} + \frac {C _ {2} P}{C _ {3}}.
$$
Let  $0 < \varepsilon \leq 1$  and choose  $\hat{\varepsilon} = G\varepsilon$ . Then the number of iterations  $T$  is
$$
\begin{array}{l} T = \frac {\lambda}{\varepsilon^ {3 / 2}} = \frac {\lambda G ^ {3 / 2}}{\hat {\varepsilon} ^ {3 / 2}} \\ = \frac {1}{\hat {\varepsilon} ^ {3 / 2} C _ {1} D ^ {3}} \left(\frac {2 C _ {1} D ^ {2} e \| \tilde {w} _ {0} - w _ {*} \| ^ {2} + C _ {2} P}{C _ {3}}\right) ^ {3 / 2} \\ = \frac {1}{\hat {\varepsilon} ^ {3 / 2}} \cdot \frac {1}{C _ {1} D ^ {3} C _ {3} ^ {3 / 2}} \left(2 C _ {1} D ^ {2} e \| \tilde {w} _ {0} - w _ {*} \| ^ {2} + C _ {2} P\right) ^ {3 / 2} \\ = \frac {1}{\hat {\varepsilon} ^ {3 / 2}} \cdot \frac {\left(2 D ^ {2} e \| \tilde {w} _ {0} - w _ {*} \| ^ {2} \left(\frac {8 L ^ {2}}{3} + \frac {1 4 N L ^ {2}}{M} + \frac {4 L ^ {2}}{3 M}\right) + \left(\frac {2 + M}{M} + \frac {5}{6 L ^ {2}} + \frac {8 N}{3 M L ^ {2}} + \frac {5}{1 2 M L}\right) P\right) ^ {3 / 2}}{\left(\frac {8 L ^ {2}}{3} + \frac {1 4 N L ^ {2}}{M} + \frac {4 L ^ {2}}{3 M}\right) D ^ {3} \left(\frac {1}{L ^ {2} + 1} \frac {\mu}{M}\right) ^ {3 / 2}} \\ \end{array}
$$
to guarantee
Hence, the total complexity (number of individual gradient computations needed to reach  $\hat{\varepsilon}$  accuracy) is  $\mathcal{O}\left(\frac{n}{\hat{\varepsilon}^{3/2}}\right)$ .
If we further assume that  $L, M, N > 1$ :
$$
\begin{array}{l} T = \frac {1}{\hat {\varepsilon} ^ {3 / 2}} \cdot \frac {\left(2 D ^ {2} e \| \tilde {w} _ {0} - w _ {*} \| ^ {2} \left(\frac {8 M L ^ {2}}{3} + 1 4 N L ^ {2} + \frac {4 L ^ {2}}{3}\right) + \left(2 + M + \frac {5 M}{6 L ^ {2}} + \frac {8 N}{3 L ^ {2}} + \frac {5}{1 2 L}\right) P\right) ^ {3 / 2}}{\left(\frac {8 L ^ {2}}{3} + \frac {1 4 N L ^ {2}}{M} + \frac {4 L ^ {2}}{3 M}\right) D ^ {3} \left(\frac {\mu}{L ^ {2} + 1}\right) ^ {3 / 2}} \\ \leq \frac {1}{\hat {\varepsilon} ^ {3 / 2}} \cdot \left(\mathcal {O} ((M + N) L ^ {2})\right) ^ {3 / 2} \cdot \mathcal {O} \left(1 / L ^ {2}\right) \cdot \left(\frac {L ^ {2} + 1}{\mu}\right) ^ {3 / 2} \\ = \mathcal {O} \left(\frac {L ^ {4} (M + N) ^ {3 / 2}}{\mu^ {3 / 2}} \cdot \frac {1}{\hat {\varepsilon} ^ {3 / 2}}\right) \\ \end{array}
$$
and the complexity is  $\mathcal{O}\left(\frac{L^4(M + N)^{3 / 2}}{\mu^{3 / 2}}\cdot \frac{n}{\hat{\varepsilon}^{3 / 2}}\right)$ .