SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \boldsymbol {G} _ {t, i} ^ {2} \leq \boldsymbol {G} _ {t - 1, i} ^ {2} + 2 | \boldsymbol {G} _ {t, i} | | \boldsymbol {G} _ {t, i} - \boldsymbol {G} _ {t - 1, i} | + 2 (\boldsymbol {G} _ {t, i} - \boldsymbol {G} _ {t - 1, i}) ^ {2} \\ \leq \boldsymbol {G} _ {t - 1, i} ^ {2} + 2 L | \boldsymbol {G} _ {t, i} | \| \boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1} \| + 2 L ^ {2} \| \boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1} \| ^ {2}, \\ \end{array}
$$
which further leads to
$$
\begin{array}{l} \frac {1}{\sqrt {\beta_ {2} \tilde {\nu} _ {t , i}}} G _ {t, i} ^ {2} \\ \leq \frac {1}{\sqrt {\beta_ {2} \widetilde {\boldsymbol {\nu}} _ {t , i}}} \left(\boldsymbol {G} _ {t - 1, i} ^ {2} + 2 L | \boldsymbol {G} _ {t, i} | \| \boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1} \| + 2 L ^ {2} \| \boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1} \| ^ {2}\right) \\ \stackrel {(\circ)} {\leq} \frac {1}{\sqrt {\beta_ {2} \widetilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {G} _ {t - 1, i} ^ {2} + \frac {(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) (1 - \beta_ {1}) \sqrt {\beta_ {2}}}{1 6 \sigma_ {1} ^ {2}} \frac {| \mathbf {G} _ {t , i} | ^ {2}}{\sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}}} + \frac {1 6 L ^ {2} \sigma_ {1} ^ {2}}{\beta_ {2} ^ {\frac {3}{2}} (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) (1 - \beta_ {1}) \sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}}} \| \mathbf {w} _ {t} - \mathbf {w} _ {t - 1} \| ^ {2} \\ + \frac {2 L ^ {2}}{\sqrt {\beta_ {2} \widetilde {\boldsymbol {\nu}} _ {t , i}}} \| \boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1} \| ^ {2} \\ \leq \frac {1}{\sqrt {\beta_ {2} \widetilde {\boldsymbol {\nu}} _ {t , i}}} \boldsymbol {G} _ {t - 1, i} ^ {2} + \frac {(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) (1 - \beta_ {1}) \sqrt {\beta_ {2}}}{1 6 \sigma_ {1} ^ {2}} \frac {| \boldsymbol {G} _ {t , i} | ^ {2}}{\sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}}} + \frac {1 6 L ^ {2} \sigma_ {1} ^ {2} \eta^ {2}}{\beta_ {2} ^ {\frac {3}{2}} (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) (1 - \beta_ {1}) \sigma_ {0} \sqrt {1 - \beta_ {2}}} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2} \\ + \frac {2 L ^ {2} \eta^ {2}}{\sigma_ {0} \sqrt {\beta_ {2} (1 - \beta_ {2})}} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2} \\ \leq \frac {1}{\sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {t , i}}} \boldsymbol {G} _ {t - 1, i} ^ {2} + \frac {(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) (1 - \beta_ {1}) \sqrt {\beta_ {2}}}{1 6 \sigma_ {1} ^ {2}} \frac {| \boldsymbol {G} _ {t , i} | ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} + \frac {1 6 (1 + \sigma_ {1} ^ {2}) L ^ {2} \eta^ {2}}{\beta_ {2} ^ {\frac {3}{2}} (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) (1 - \beta_ {1}) \sigma_ {0} \sqrt {1 - \beta_ {2}}} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2}. \\ \end{array}
$$
Applying the above inequality back to Eq. (11) leads to that
$$
\begin{array}{l} \text {I . 1 . 1} = \sum_ {i = 1} ^ {d} \frac {\eta}{1 - \beta_ {1}} \mathbb {E} ^ {| \mathcal {F} _ {t}} \left[ | \boldsymbol {G} _ {t, i} | \left(\frac {(1 - \beta_ {2}) \boldsymbol {g} _ {t , i} ^ {2}}{\sqrt {\boldsymbol {\nu} _ {t , i}} \sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}} (\sqrt {\boldsymbol {\nu} _ {t , i}} + \sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}})}\right) | \boldsymbol {m} _ {t, i} | \right] \\ \leq \frac {1}{2} \sum_ {i = 1} ^ {d} \eta \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \frac {\left| \boldsymbol {G} _ {t , i} \right| ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} + \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{\left(1 - \frac {\beta_ {1} ^ {2}}{\beta_ {2}}\right) ^ {2}} \sum_ {i = 1} ^ {d} _ {\mathbb {E}} ^ {| \mathcal {F} _ {t}} \frac {\boldsymbol {g} _ {t , i} ^ {2}}{\boldsymbol {\nu} _ {t , i}} \\ + \eta \frac {4 (1 - \beta_ {1})}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} \sqrt {\beta_ {2}}} \sigma_ {1} ^ {2} \sum_ {i = 1} ^ {d} \mathbb {E} ^ {| \mathcal {F} _ {t}} \left(\frac {\boldsymbol {G} _ {t - 1 , i} ^ {2}}{\sqrt {\beta_ {2} \tilde {\nu} _ {t , i}}} - \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\tilde {\nu} _ {t + 1 , i}}}\right) \\ + \frac {6 4 d \left(1 + \sigma_ {1} ^ {2}\right) \sigma_ {1} ^ {2} L ^ {2} \eta^ {3}}{\beta_ {2} ^ {2} \left(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}\right) ^ {3} \left(1 - \beta_ {1}\right) \sigma_ {0} \sqrt {1 - \beta_ {2}}} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2}. \tag {12} \\ \end{array}
$$
As for I.1.2, we have
$$
\begin{array}{l} \text {I .} 1. 2 = \sum_ {i = 1} ^ {d} \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \mathbb {E} ^ {| \mathcal {F} _ {t}} \left[ | \boldsymbol {G} _ {t, i} | \left(\frac {(1 - \beta_ {2}) \sigma_ {0} ^ {2}}{\sqrt {\boldsymbol {\nu} _ {t , i}} \sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}} (\sqrt {\boldsymbol {\nu} _ {t , i}} + \sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}})}\right) | \boldsymbol {m} _ {t, i} | \right] \\ \leq \sum_ {i = 1} ^ {d} \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \mathbb {E} ^ {| \mathcal {F} _ {t}} \left[ | \boldsymbol {G} _ {t, i} | \left(\frac {\sqrt [ 4 ]{1 - \beta_ {2}} \sqrt {\sigma_ {0}}}{\sqrt [ 4 ]{\tilde {\nu} _ {t , i}} \sqrt {\nu_ {t , i}}}\right) | \boldsymbol {m} _ {t, i} | \right] \\ \leq \frac {1 - \beta_ {1}}{8 \left(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}\right)} \sum_ {i = 1} ^ {d} \eta \frac {\left| G _ {t , i} \right| ^ {2}}{\sqrt {\tilde {\nu} _ {t , i}}} + \sum_ {i = 1} ^ {d} \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{\left(1 - \beta_ {1}\right) \left(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}\right)} \mathbb {E} ^ {| \mathcal {F} _ {t}} \left[ \left(\frac {\left| m _ {t , i} \right| ^ {2}}{\nu_ {t , i}}\right) \right]. \tag {13} \\ \end{array}
$$
With Inequalities (12) and (13), we conclude that
$$
\begin{array}{l} \mathrm {I}. 1 \leq \frac {5}{8} \sum_ {i = 1} ^ {d} \eta \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \mathbb {E} \frac {| \boldsymbol {G} _ {t , i} | ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} + \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{\left(1 - \frac {\beta_ {1} ^ {2}}{\beta_ {2}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {g} _ {t , i} ^ {2}}{\boldsymbol {\nu} _ {t , i}} \\ + \eta \frac {4 (1 - \beta_ {1})}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} \sqrt {\beta_ {2}}} \sigma_ {1} ^ {2} \sum_ {i = 1} ^ {d} \mathbb {E} \left(\frac {\boldsymbol {G} _ {t - 1 , i} ^ {2}}{\sqrt {\beta_ {2} \tilde {\nu} _ {t , i}}} - \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\tilde {\nu} _ {t + 1 , i}}}\right) + \sum_ {i = 1} ^ {d} \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{(1 - \beta_ {1}) (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}})} \mathbb {E} \left[ \left(\frac {| \boldsymbol {m} _ {t , i} | ^ {2}}{\boldsymbol {\nu} _ {t , i}}\right) \right] \\ + \frac {6 4 (1 + \sigma_ {1} ^ {2}) \sigma_ {1} ^ {2} L ^ {2} \eta^ {3} d}{\beta_ {2} ^ {2} (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {3} (1 - \beta_ {1}) \sigma_ {0} \sqrt {1 - \beta_ {2}}} \mathbb {E} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2}. \\ \end{array}
$$
Lemma 8 (Formal version of Lemma 2). Let all conditions in Theorem 1 hold. Then,
$$
\begin{array}{l} \mathbb {E} f \left(\boldsymbol {u} _ {t + 1}\right) \\ \leq \mathbb {E} f (\boldsymbol {u} _ {t}) - \frac {\eta}{4} \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \mathbb {E} \left[ \eta \left\langle \boldsymbol {G} _ {t}, - \frac {1}{\sqrt {\widetilde {\boldsymbol {\nu}} _ {t}}} \odot \boldsymbol {G} _ {t} \right\rangle \right] + \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{\left(1 - \frac {\beta_ {1} ^ {2}}{\beta_ {2}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {g} _ {t , i} ^ {2}}{\boldsymbol {\nu} _ {t , i}} \\ + \eta \frac {4}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} \sqrt {\beta_ {2}}} \sigma_ {1} ^ {2} \sum_ {i = 1} ^ {d} \mathbb {E} \left(\frac {1}{\sqrt {\beta_ {2}}} \xi_ {t - 1} - \xi_ {t}\right) + \sum_ {i = 1} ^ {d} \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{(1 - \beta_ {1}) (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}})} \mathbb {E} \left[ \left(\frac {| m _ {t , i} | ^ {2}}{\boldsymbol {\nu} _ {t , i}}\right) \right] \\ + \frac {6 4 (1 + \sigma_ {1} ^ {2}) \sigma_ {1} ^ {2} L ^ {2} \eta^ {3} d}{\beta_ {2} ^ {2} (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {3} (1 - \beta_ {1}) \sigma_ {0} \sqrt {1 - \beta_ {2}}} \mathbb {E} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2} + \frac {2 \eta \sqrt {1 - \beta_ {2}} \beta_ {1} ^ {2} \sigma_ {0}}{(1 - \beta_ {1}) (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) \beta_ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \left[ \frac {| \boldsymbol {m} _ {t - 1 , i} | ^ {2}}{\boldsymbol {\nu} _ {t - 1 , i}} \right] \\ + L \mathbb {E} \left[ 4 \left(\frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \eta^ {2} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2} + 3 \left(\frac {1}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \eta^ {2} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} \odot \boldsymbol {m} _ {t} \right\| ^ {2} \right]. \\ \end{array}
$$
Proof. According to the definition of  $\pmb{u}_t$ , we have
$$
\begin{array}{l} \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} = \frac {\boldsymbol {w} _ {t + 1} - \boldsymbol {w} _ {t}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}} \frac {\boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \\ = - \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} \odot \boldsymbol {m} _ {t} + \beta_ {1} \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \frac {1}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \\ = - \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}} \odot \boldsymbol {m} _ {t} + \beta_ {1} \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}} \odot \boldsymbol {m} _ {t - 1} \\ - \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \left(\frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} - \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}}\right) \odot \boldsymbol {m} _ {t} + \beta_ {1} \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \left(\frac {1}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1}}} - \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}}\right) \odot \boldsymbol {m} _ {t - 1} \\ \stackrel {(*)} {=} - \eta \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}} \odot \boldsymbol {g} _ {t} \\ - \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \left(\frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} - \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}}\right) \odot \boldsymbol {m} _ {t} + \beta_ {1} \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \left(\frac {1}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1}}} - \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}}\right) \odot \boldsymbol {m} _ {t - 1}, \\ \end{array}
$$
where Eq.  $(\ast)$  is due to  $\pmb{m}_t = \beta_1\pmb{m}_{t - 1} + (1 - \beta_1)\pmb{g}_t$
Applying the above equation to the "First Order" term, we find that it can be decomposed as
$$
\begin{array}{l} \mathbb {E} \left[ \langle \nabla f (\boldsymbol {u} _ {t}), \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} \rangle \right] \\ = \mathbb {E} \left[ \left\langle \boldsymbol {G} _ {t}, \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} \right\rangle \right] + \mathbb {E} \left[ \left\langle \nabla f (\boldsymbol {u} _ {t}) - G _ {t}, \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} \right\rangle \right] \\ = \mathbb {E} \left[ \left\langle \boldsymbol {G} _ {t}, - \eta \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}} \odot \boldsymbol {g} _ {t} \right\rangle \right] + \mathbb {E} \left[ \left\langle \boldsymbol {G} _ {t}, - \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \left(\frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} - \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}}\right) \odot \boldsymbol {m} _ {t} \right\rangle \right] \\ + \mathbb {E} \left[ \left\langle \boldsymbol {G} _ {t}, \beta_ {1} \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \left(\frac {1}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1}}} - \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}}\right) \odot \boldsymbol {m} _ {t - 1} \right\rangle \right] + \mathbb {E} \left[ \left\langle \nabla f (\boldsymbol {u} _ {t}) - \boldsymbol {G} _ {t}, \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} \right\rangle \right] \\ = - \eta \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \mathbb {E} \left\| \frac {1}{\sqrt [ 4 ]{\tilde {\boldsymbol {\nu}} _ {t}}} \odot \boldsymbol {G} _ {t} \right\| ^ {2} + \underbrace {\mathbb {E} \left[ \left\langle \boldsymbol {G} _ {t} , - \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \left(\frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} - \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}}\right) \odot \boldsymbol {m} _ {t} \right\rangle \right]} _ {\text {I . I}} \\ + \underbrace {\mathbb {E} \left[ \left\langle \boldsymbol {G} _ {t} , \beta_ {1} \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \left(\frac {1}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1}}} - \frac {1}{\sqrt {\widetilde {\boldsymbol {\nu}} _ {t}}}\right) \odot \boldsymbol {m} _ {t - 1} \right\rangle \right]} _ {\text {I . 2}} + \underbrace {\mathbb {E} \left[ \left\langle \nabla f (\boldsymbol {u} _ {t}) - \boldsymbol {G} _ {t} , \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} \right\rangle \right]} _ {\text {I . 3}}. \\ \end{array}
$$
Here we apply Lemma 7 to bound I.1. We proceed by bounding I.2 and I.3 respectively.
As for I.2, we have
$$
\begin{array}{l} I. 2 = \mathbb {E} \left[ \left\langle \boldsymbol {G} _ {t}, \beta_ {1} \frac {\eta}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \left(\frac {1}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1}}} - \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t}}}\right) \odot \boldsymbol {m} _ {t - 1} \right\rangle \right] \\ \leq \frac {\eta \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {i = 1} ^ {d} \mathbb {E} \left[ | \boldsymbol {G} _ {t, i} | \left| \frac {1}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1 , i}}} - \frac {1}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \right| | \boldsymbol {m} _ {t - 1, i} | \right] \\ = \frac {\eta \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {i = 1} ^ {d} \mathbb {E} \left[ | \boldsymbol {G} _ {t, i} | \left| \frac {(1 - \beta_ {2}) \sigma_ {0} ^ {2}}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1 , i}} \sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}} (\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}} + \sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1 , i}})} \right| | \boldsymbol {m} _ {t - 1, i} | \right] \\ = \frac {\eta \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {i = 1} ^ {d} \mathbb {E} \left[ | \boldsymbol {G} _ {t, i} | \left| \frac {\sqrt [ 4 ]{1 - \beta_ {2}} \sqrt {\sigma_ {0}}}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1 , i}} \sqrt [ 4 ]{\widetilde {\boldsymbol {\nu}} _ {t , i}}} \right| | \boldsymbol {m} _ {t - 1, i} | \right] \\ \leq \frac {1}{8} \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {i = 1} ^ {d} \eta \mathbb {E} \frac {| \boldsymbol {G} _ {t , i} | ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} + \frac {2 \eta \sqrt {1 - \beta_ {2}} \beta_ {1} ^ {2} \sigma_ {0}}{(1 - \beta_ {1}) (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) \beta_ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \left[ \frac {| \boldsymbol {m} _ {t - 1 , i} | ^ {2}}{\boldsymbol {\nu} _ {t - 1 , i}} \right]. \\ \end{array}
$$
$$
\begin{array}{l} I. 3 = \mathbb {E} \left[ \left\langle \nabla f \left(\boldsymbol {u} _ {t}\right) - \boldsymbol {G} _ {t}, \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} \right\rangle \right] \\ \leq \mathbb {E} \left[ \| \nabla f (\boldsymbol {u} _ {t}) - \boldsymbol {G} _ {t} \| \| \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} \| \right] \\ \leq L \mathbb {E} \left[ \| \boldsymbol {u} _ {t} - \boldsymbol {w} _ {t} \| \| \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} \| \right] \\ = L \mathbb {E} \left[ \frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \| \boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1} \| \left(\frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \| \boldsymbol {w} _ {t + 1} - \boldsymbol {w} _ {t} \| + \frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \| \boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1} \|\right) \right] \\ \leq L \mathbb {E} \left[ \frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \| \boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1} \| \left(\frac {1}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \| \boldsymbol {w} _ {t + 1} - \boldsymbol {w} _ {t} \| + \frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \| \boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1} \|\right) \right] \\ \leq L \mathbb {E} \left[ 2 \left(\frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \| \pmb {w} _ {t} - \pmb {w} _ {t - 1} \| ^ {2} + \frac {1}{4} \left(\frac {1}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \| \pmb {w} _ {t + 1} - \pmb {w} _ {t} \| ^ {2} \right] \\ \leq L \mathbb {E} \left[ 2 \left(\frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \eta^ {2} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2} + \frac {1}{4} \left(\frac {1}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \eta^ {2} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} \odot \boldsymbol {m} _ {t} \right\| ^ {2} \right]. \\ \end{array}
$$
All in all, we summarize that the "First Order" term can be bounded by
$$
\begin{array}{l} - \frac {\eta}{4} \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\tilde {\nu} _ {t}}}} \mathbb {E} \left\| \frac {1}{\sqrt [ 4 ]{\tilde {\nu} _ {t}}} \odot \boldsymbol {G} _ {t} \right\| ^ {2} + \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{\left(1 - \frac {\beta_ {1} ^ {2}}{\beta_ {2}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {g} _ {t , i} ^ {2}}{\boldsymbol {\nu} _ {t , i}} \\ + \eta \frac {4 (1 - \beta_ {1})}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} \sqrt {\beta_ {2}}} \sigma_ {1} ^ {2} \sum_ {i = 1} ^ {d} \mathbb {E} \left(\frac {\pmb {G} _ {t - 1 , i} ^ {2}}{\sqrt {\beta_ {2} \widetilde {\pmb {\nu}} _ {t , i}}} - \frac {\pmb {G} _ {t , i} ^ {2}}{\sqrt {\widetilde {\pmb {\nu}} _ {t + 1 , i}}}\right) + \sum_ {i = 1} ^ {d} \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{(1 - \beta_ {1}) (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}})} \mathbb {E} \left[ \left(\frac {| \pmb {m} _ {t , i} | ^ {2}}{\pmb {\nu} _ {t , i}}\right) \right] \\ + \frac {6 4 (1 + \sigma_ {1} ^ {2}) \sigma_ {1} ^ {2} L ^ {2} \eta^ {3} d}{\beta_ {2} ^ {2} (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {3} (1 - \beta_ {1}) \sigma_ {0} \sqrt {1 - \beta_ {2}}} \mathbb {E} \left\| \frac {1}{\sqrt {\pmb {\nu} _ {t - 1}}} \odot \pmb {m} _ {t - 1} \right\| ^ {2} + \frac {2 \eta \sqrt {1 - \beta_ {2}} \beta_ {1} ^ {2} \sigma_ {0}}{(1 - \beta_ {1}) (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) \beta_ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \left[ \frac {| \pmb {m} _ {t - 1 , i} | ^ {2}}{\pmb {\nu} _ {t - 1 , i}} \right] \\ + L \mathbb {E} \left[ 2 \left(\frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \eta^ {2} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2} + \frac {1}{4} \left(\frac {1}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \eta^ {2} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} \odot \boldsymbol {m} _ {t} \right\| ^ {2} \right]. \\ \end{array}
$$
Furthermore, the "Second Order" term can be directly bounded by
$$
\begin{array}{l} \frac {L}{2} \mathbb {E} \| \boldsymbol {u} _ {t + 1} - \boldsymbol {u} _ {t} \| ^ {2} = \frac {L}{2} \left\| \frac {\boldsymbol {w} _ {t + 1} - \boldsymbol {w} _ {t}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}} \frac {\boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \right\| ^ {2} \\ \leq 2 L \mathbb {E} \left\| \frac {\boldsymbol {w} _ {t + 1} - \boldsymbol {w} _ {t}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \right\| ^ {2} + 2 L \mathbb {E} \left\| \frac {\beta_ {1}}{\sqrt {\beta_ {2}}} \frac {\boldsymbol {w} _ {t} - \boldsymbol {w} _ {t - 1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \right\| ^ {2}. \\ \end{array}
$$
As for I.3, we directly apply Assumption 1 and obtain
Applying the estimations of the first-order term and the second-order term to the descent lemma then gives
$$
\begin{array}{l} \mathbb {E} f (\boldsymbol {u} _ {t + 1}) \\ \leq \mathbb {E} f (\boldsymbol {u} _ {t}) - \frac {\eta}{4} \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} + \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{\left(1 - \frac {\beta_ {1} ^ {2}}{\beta_ {2}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {g} _ {t , i} ^ {2}}{\boldsymbol {\nu} _ {t , i}} \\ + \eta \frac {4}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} \sqrt {\beta_ {2}}} \sigma_ {1} ^ {2} \sum_ {i = 1} ^ {d} \mathbb {E} \left(\frac {\boldsymbol {G} _ {t - 1 , i} ^ {2}}{\sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {t , i}}} - \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t + 1 , i}}}\right) + \sum_ {i = 1} ^ {d} \frac {2 \eta \sqrt {1 - \beta_ {2}} \sigma_ {0}}{(1 - \beta_ {1}) (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}})} \mathbb {E} \left[ \left(\frac {| \boldsymbol {m} _ {t , i} | ^ {2}}{\boldsymbol {\nu} _ {t , i}}\right) \right] \\ + \frac {6 4 (1 + \sigma_ {1} ^ {2}) \sigma_ {1} ^ {2} L ^ {2} \eta^ {3} d}{\beta_ {2} ^ {2} (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {3} (1 - \beta_ {1}) \sigma_ {0} \sqrt {1 - \beta_ {2}}} \mathbb {E} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2} + \frac {2 \eta \sqrt {1 - \beta_ {2}} \beta_ {1} ^ {2} \sigma_ {0}}{(1 - \beta_ {1}) (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) \beta_ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \left[ \frac {| \boldsymbol {m} _ {t - 1 , i} | ^ {2}}{\boldsymbol {\nu} _ {t - 1 , i}} \right] \\ + L \mathbb {E} \left[ 4 \left(\frac {\frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \eta^ {2} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t - 1}}} \odot \boldsymbol {m} _ {t - 1} \right\| ^ {2} + 3 \left(\frac {1}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} \eta^ {2} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} \odot \boldsymbol {m} _ {t} \right\| ^ {2} \right]. \\ \end{array}
$$
Lemma 9 (Lemma 3, restated). Let all conditions in Theorem 1 hold. Then,
$$
\sum_ {t = 1} ^ {T + 1} \sum_ {i = 1} ^ {d} \mathbb {E} \sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}} \leq 2 (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\boldsymbol {\nu} _ {0 , i} + \sigma_ {0} ^ {2}} + 2 4 d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} \ln d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} + \frac {1 2 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} C _ {2}.
$$
Proof of Lemma 3. To begin with, we have that