SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\sum_ {t = 1} ^ {T} \mathbb {E} \left[ \frac {\left| \boldsymbol {G} _ {t , i} \right| ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {\left| G _ {t, i} \right| \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \right] \leq \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \frac {\left| \boldsymbol {G} _ {t , i} \right| ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \right]. \tag {14}
$$
$$
\begin{array}{l} \frac {\left| \boldsymbol {G} _ {t , i} \right| ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \geq \frac {\frac {2}{3} \left| \boldsymbol {G} _ {t , i} \right| ^ {2} + \frac {1}{3} \frac {\sigma_ {0} ^ {2}}{\sigma_ {1} ^ {2}}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \geq \frac {\frac {\beta_ {2}}{3 \sigma_ {1} ^ {2}} \mathbb {E} ^ {| \mathcal {F} _ {t} |} \left| \boldsymbol {g} _ {t , i} \right| ^ {2} + \frac {1 - \beta_ {2}}{3} \frac {\sigma_ {0} ^ {2}}{\sigma_ {1} ^ {2}}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \\ = \mathbb {E} ^ {| \mathcal {F} _ {t}} \frac {\frac {\beta_ {2}}{3 \sigma_ {1} ^ {2}} | \boldsymbol {g} _ {t , i} | ^ {2} + \frac {1 - \beta_ {2}}{3 \sigma_ {1} ^ {2}} \sigma_ {0} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \geq \sqrt {\beta_ {2}} \mathbb {E} ^ {| \mathcal {F} _ {t}} \frac {\frac {\beta_ {2}}{3 \sigma_ {1} ^ {2}} | \boldsymbol {g} _ {t , i} | ^ {2} + \frac {1 - \beta_ {2}}{3 \sigma_ {1} ^ {2}} \sigma_ {0} ^ {2}}{\sqrt {\tilde {\boldue q} _ {t + 1 , i}} + \sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}}. \\ \end{array}
$$
As a conclusion,
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \frac {\left| \boldsymbol {G} _ {t , i} \right| ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {\left| G _ {t, i} \right| \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \right] \geq \sqrt {\beta_ {2}} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \frac {\frac {\beta_ {2}}{3 \sigma_ {1} ^ {2}} \left| \boldsymbol {g} _ {t , i} \right| ^ {2} + \frac {1 - \beta_ {2}}{3 \sigma_ {1} ^ {2}} \sigma_ {0} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t + 1 , i}} + \sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {\left| G _ {t, i} \right| \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \right] \\ \geq \frac {\sqrt {\beta_ {2}}}{3 (1 - \beta_ {2}) \sigma_ {1} ^ {2}} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \left(\sqrt {\tilde {\boldsymbol {\nu}} _ {t + 1 , i}} - \sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {t , i}}\right) \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \right]. \\ \end{array}
$$
On the other hand, as stated in Section 4.2, we define  $\{\bar{\nu}_{t,i}\}_{t=0}^{\infty}$  as  $\bar{\nu}_{0,i} = \nu_{0,i}$ ,  $\bar{\nu}_{t,i} = \beta_2\bar{\nu}_{t-1,i} + (1 - \beta_2)|g_{t,i}|^2\mathbf{1}_{|\pmb{G}_{t,i}| < \frac{\sigma_0^2}{\sigma_1^2}}$ . One can easily observe that  $\bar{\nu}_{t,i} \leq \nu_{t,i}$ , and thus
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \left(\sqrt {\tilde {\boldsymbol {\nu}} _ {t + 1 , i}} - \sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {t , i}}\right) \mathbf {1} _ {| \boldsymbol {G} _ {t, i} | <   \frac {\sigma_ {0} ^ {2}}{\sigma_ {1} ^ {2}}} \right] \\ = \sum_ {t = 1} ^ {T} \mathbb {E} \left(\sqrt {\beta_ {2} ^ {2} \boldsymbol {\nu} _ {t - 1 , i} + \beta_ {2} (1 - \beta_ {2}) | g _ {t , i} | ^ {2} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} - \sqrt {\beta_ {2} (\beta_ {2} \boldsymbol {\nu} _ {t - 1 , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2})}\right) \mathbf {1} _ {| \boldsymbol {G} _ {t, i} | <   \frac {\sigma_ {0} ^ {2}}{\sigma_ {1} ^ {2}}} \\ \leq \sum_ {t = 1} ^ {T} \mathbb {E} \left(\sqrt {\beta_ {2} ^ {2} \bar {\nu} _ {t - 1 , i} + \beta_ {2} (1 - \beta_ {2}) | g _ {t , i} | ^ {2} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} - \sqrt {\beta_ {2} (\beta_ {2} \bar {\nu} _ {t - 1 , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2})}\right) \mathbf {1} _ {| G _ {t, i} | <   \frac {\sigma_ {0} ^ {2}}{\sigma_ {1} ^ {2}}} \\ \leq \sum_ {t = 1} ^ {T} \mathbb {E} \left(\sqrt {\beta_ {2} ^ {2} \bar {\nu} _ {t - 1 , i} + \beta_ {2} (1 - \beta_ {2}) | g _ {t , i} | ^ {2} \mathbf {1} _ {| \boldsymbol {G} _ {t , i} | <   \frac {\sigma_ {0} ^ {2}}{\sigma_ {1} ^ {2}} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}}} - \sqrt {\beta_ {2} (\beta_ {2} \bar {\nu} _ {t - 1 , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2})}\right) \\ = \sum_ {t = 1} ^ {T} \mathbb {E} \left(\sqrt {\beta_ {2} \bar {\nu} _ {t , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} - \sqrt {\beta_ {2} \left(\beta_ {2} \bar {\nu} _ {t - 1 , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}\right)}\right) \\ = \mathbb {E} \sqrt {\beta_ {2} \bar {\nu} _ {t , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} + (1 - \sqrt {\beta_ {2}}) \sum_ {t = 1} ^ {T - 1} \mathbb {E} \sqrt {\beta_ {2} \bar {\nu} _ {t , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} - \mathbb {E} \sqrt {\beta_ {2} (\beta_ {2} \bar {\nu} _ {0 , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2})}. \\ \end{array}
$$
All in all, summing the above two inequalities together, we obtain that
$$
\begin{array}{l} \mathbb {E} \sqrt {\widetilde {\boldsymbol {\nu}} _ {t + 1 , i}} + (1 - \sqrt {\beta_ {2}}) \sum_ {t = 2} ^ {T} \mathbb {E} \sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}} - \sqrt {\beta_ {2} \widetilde {\boldsymbol {\nu}} _ {1 , i}} \\ = \sum_ {t = 1} ^ {T} \mathbb {E} \left(\sqrt {\tilde {\nu} _ {t , i}} - \sqrt {\beta_ {2} \tilde {\nu} _ {t - 1 , i}}\right) \\ \leq \sum_ {t = 1} ^ {T} \mathbb {E} \left(\sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}} - \sqrt {\beta_ {2} \widetilde {\boldsymbol {\nu}} _ {t - 1 , i}}\right) \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}} + \sum_ {t = 1} ^ {T} \mathbb {E} \left(\sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}} - \sqrt {\beta_ {2} \widetilde {\boldsymbol {\nu}} _ {t - 1 , i}}\right) \mathbf {1} _ {| G _ {t, i} | <   \frac {\sigma_ {0} ^ {2}}{\sigma_ {1} ^ {2}}} \\ \leq \frac {3 (1 - \beta_ {2}) \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \frac {| \boldsymbol {G} _ {t , i} | ^ {2}}{\sqrt {\tilde {\nu} _ {t , i}}} \right] + \mathbb {E} \sqrt {\beta_ {2} \bar {\nu} _ {t , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} + (1 - \sqrt {\beta_ {2}}) \sum_ {t = 1} ^ {T - 1} \mathbb {E} \sqrt {\beta_ {2} \bar {\nu} _ {t , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} - \sqrt {\beta_ {2} (\beta_ {2} \bar {\nu} _ {0 , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2})}. \\ \end{array}
$$
Since  $\forall t$
$$
\mathbb {E} \sqrt {\beta_ {2} \bar {\nu} _ {t , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} \leq \sqrt {\beta_ {2} \mathbb {E} \bar {\nu} _ {t , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} \leq \sqrt {\sigma_ {0} ^ {2} + \boldsymbol {\nu} _ {0 , i}},
$$
combining with  $\sqrt{\beta_2\tilde{\nu}_{1,i}} = \sqrt{\beta_2(\beta_2\bar{\nu}_{0,i} + (1 - \beta_2)\sigma_0^2)}$  and  $\mathbb{E}\sqrt{\tilde{\nu}_{t+1,i}} = \mathbb{E}\sqrt{\beta_2\bar{\nu}_{t,i} + (1 - \beta_2)\sigma_0^2}$ , we obtain
$$
\begin{array}{l} (1 - \sqrt {\beta_ {2}}) \sum_ {t = 2} ^ {T + 1} \mathbb {E} \sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}} \leq \frac {3 (1 - \beta_ {2}) \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} \sum_ {t = 2} ^ {T} \mathbb {E} \left[ \frac {| \boldsymbol {G} _ {t , i} | ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \right] + + (1 - \sqrt {\beta_ {2}}) \sum_ {t = 1} ^ {T} \mathbb {E} \sqrt {\beta_ {2} \bar {\boldsymbol {\nu}} _ {t , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}} \\ \leq \frac {3 \left(1 - \beta_ {2}\right) \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} \sum_ {t = 2} ^ {T} \mathbb {E} \left[ \frac {\left| \boldsymbol {G} _ {t , i} \right| ^ {2}}{\sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}}} \right] + (1 - \sqrt {\beta_ {2}}) T \sqrt {\sigma_ {0} ^ {2} + \boldsymbol {\nu} _ {0 , i}}. \tag {15} \\ \end{array}
$$
Leveraging Eq. (16), we then obtain that
$$
\begin{array}{l} \sum_ {t = 1} ^ {T + 1} \sum_ {i = 1} ^ {d} \mathbb {E} \sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}} \\ \leq 3 \frac {\left(1 + \sqrt {\beta_ {2}}\right) \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \frac {\left| \boldsymbol {G} _ {t , i} \right| ^ {2}}{\sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}}} \right] + (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\boldsymbol {\nu} _ {0 , i} + \sigma_ {0} ^ {2}} \\ \leq \frac {6 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} \left(\frac {1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \beta_ {1}} \frac {8}{\eta} f (\boldsymbol {u} _ {1}) + \frac {3 2}{\beta_ {2} \left(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {1 , i} ^ {2}}{\sqrt {\tilde {\nu} _ {1 , i}}} + C _ {1} \mathbb {E} \sum_ {i = 1} ^ {d} \left(\ln \left(\frac {\boldsymbol {\nu} _ {T , i}}{\boldsymbol {\nu} _ {0 , i}}\right) - T \ln \beta_ {2}\right)\right) \\ + (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\nu_ {0 , i} + \sigma_ {0} ^ {2}} \\ \leq \frac {6 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} \left(\frac {1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \beta_ {1}} \frac {8}{\eta} f (\boldsymbol {u} _ {1}) + \frac {3 2}{\beta_ {2} \left(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {1 , i} ^ {2}}{\sqrt {\tilde {\nu} _ {1 , i}}} + 2 C _ {1} \mathbb {E} \sum_ {i = 1} ^ {d} \left(\ln \left(\frac {\sum_ {t = 1} ^ {T + 1} \sqrt {\tilde {\nu} _ {t , i}}}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {0 , i}}}\right) - T \ln \beta_ {2}\right)\right) \\ + (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\nu_ {0 , i} + \sigma_ {0} ^ {2}} \\ \leq \frac {6 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} \left(\frac {1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \beta_ {1}} \frac {8}{\eta} f (\boldsymbol {u} _ {1}) + \frac {3 2}{\beta_ {2} \left(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {1 , i} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {1 , i}}} + 2 C _ {1} \sum_ {i = 1} ^ {d} \left(\ln \left(\frac {\mathbb {E} \sum_ {t = 1} ^ {T + 1} \sum_ {j = 1} ^ {d} \sqrt {\tilde {\boldsymbol {\nu}} _ {t , j}}}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {0 , i}}}\right) - T \ln \beta_ {2}\right)\right) \\ + (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\nu_ {0 , i} + \sigma_ {0} ^ {2}}, \\ \end{array}
$$
where in the last inequality we use the concavity of  $h(x) = \ln x$ . Solving the above inequality with respect to  $\sum_{t=1}^{T+1} \sum_{i=1}^{d} \mathbb{E} \sqrt{\widetilde{\pmb{\nu}}_{t,i}}$  then gives
$$
\begin{array}{l} \sum_ {t = 1} ^ {T + 1} \sum_ {i = 1} ^ {d} \mathbb {E} \sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}} \leq 2 (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\boldsymbol {\nu} _ {0 , i} + \sigma_ {0} ^ {2}} + 2 4 d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} \ln d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} \\ + \frac {1 2 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} \left(\frac {1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \beta_ {1}} \frac {8}{\eta} f (\boldsymbol {u} _ {1}) + \frac {3 2}{\beta_ {2} \left(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {1 , i} ^ {2}}{\sqrt {\widetilde {\boldsymbol {\nu}} _ {1 , i}}} + 2 C _ {1} \sum_ {i = 1} ^ {d} \left(\ln \left(\frac {1}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {0 , i}}}\right) - T \ln \beta_ {2}\right)\right). \\ \end{array}
$$
The proof is then completed by applying the definition of  $C_2$ .
C.2 Proof of Theorem 1
Proof of Theorem 1. Summing the inequality in Lemma 8 over  $t$  from 1 to  $T$  and collecting the terms, we obtain
$$
\mathbb {E} f (\boldsymbol {u} _ {T + 1})
$$
$$
\begin{array}{l} \leq f (\boldsymbol {u} _ {1}) - \frac {\eta}{4} \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {t = 1} ^ {T} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} + \eta \frac {4 (1 - \beta_ {1})}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} \sqrt {\beta_ {2}}} \sigma_ {1} ^ {2} \sum_ {t = 1} ^ {T} \sum_ {i = 1} ^ {d} \mathbb {E} \left(\frac {\boldsymbol {G} _ {t - 1 , i} ^ {2}}{\sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {t , i}}} - \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t + 1 , i}}}\right) \\ + \tilde {C} \sum_ {t = 1} ^ {T} \mathbb {E} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} \odot \boldsymbol {m} _ {t} \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \leq f (\boldsymbol {u} _ {1}) - \frac {\eta}{4} \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {t = 1} ^ {T} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} + \eta \frac {4 (1 - \beta_ {1})}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} \sqrt {\beta_ {2}}} \sigma_ {1} ^ {2} \sum_ {i = 1} ^ {d} \mathbb {E} \left(\frac {\boldsymbol {G} _ {1 , i} ^ {2}}{\sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {1 , i}}} \right. \\ \left. + \left(\frac {1}{\beta_ {2}} - 1\right) \sum_ {t = 1} ^ {T - 1} \mathbb {E} \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {t + 1 , i}}}\right) + \tilde {C} \sum_ {t = 1} ^ {T} \mathbb {E} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} \odot \boldsymbol {m} _ {t} \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(*)} {\leq} f (\boldsymbol {u} _ {1}) - \frac {\eta}{4} \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {t = 1} ^ {T} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} + \eta \frac {4 (1 - \beta_ {1})}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} \sqrt {\beta_ {2}}} \sigma_ {1} ^ {2} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {1 , i} ^ {2}}{\sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {1 , i}}} \\ + \frac {\eta}{8} \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {t = 1} ^ {T - 1} \mathbb {E} \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\widetilde {\boldsymbol {\nu}} _ {t , i}}} + \tilde {C} \sum_ {t = 1} ^ {T} \mathbb {E} \left\| \frac {1}{\sqrt {\boldsymbol {\nu} _ {t}}} \odot \boldsymbol {m} _ {t} \right\| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(\circ)} {\leq} f (\boldsymbol {u} _ {1}) - \frac {\eta}{8} \frac {1 - \beta_ {1}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}} \sum_ {t = 1} ^ {T} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {t , i} ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} + \eta \frac {4 (1 - \beta_ {1})}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} \sqrt {\beta_ {2}}} \sigma_ {1} ^ {2} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {1 , i} ^ {2}}{\sqrt {\beta_ {2} \tilde {\boldsymbol {\nu}} _ {1 , i}}} \\ + \tilde {C} \frac {(1 - \beta_ {1}) ^ {2}}{(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {2} (1 - \beta_ {2})} \sum_ {i = 1} ^ {d} \left(\ln \left(\frac {\boldsymbol {\nu} _ {T , i}}{\boldsymbol {\nu} _ {0 , i}}\right) - T \ln \beta_ {2}\right), \\ \end{array}
$$
$$
\tilde {C} \triangleq 4 L \eta^ {2} \left(\frac {1 + \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}\right) ^ {2} + \frac {2 \eta \sqrt {1 - \beta_ {2}} \beta_ {1} ^ {2} \sigma_ {0}}{(1 - \beta_ {1}) (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) \beta_ {2}} + \frac {6 4 (1 + \sigma_ {1} ^ {2}) \sigma_ {1} ^ {2} L ^ {2} \eta^ {3} d}{\beta_ {2} ^ {2} (1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}) ^ {3} (1 - \beta_ {1}) \sigma_ {0} \sqrt {1 - \beta_ {2}}}.
$$
to simplify the notations, inequality  $(*)$  is due to that  $\tilde{\nu}_{t+1,i} \geq \beta_2 \tilde{\nu}_{t+1,i}$  and  $\beta_1 \leq \sqrt{\beta_2} - 8\sigma_1^2 (1 - \beta_2)\beta_2^{-2}$ , and inequality  $(\circ)$  is due to Lemma 5. Simple rearrangement of the above inequality then gives
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \left\| \frac {1}{\sqrt [ 4 ]{\widetilde {\boldsymbol {\nu}} _ {t}}} \odot \boldsymbol {G} _ {t} \right\| ^ {2} \right] \leq \frac {1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \beta_ {1}} \frac {8}{\eta} f (\boldsymbol {u} _ {1}) + \frac {3 2}{\beta_ {2} \left(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {1 , i} ^ {2}}{\sqrt {\widetilde {\boldsymbol {\nu}} _ {1 , i}}} \\ + C _ {1} \sum_ {i = 1} ^ {d} \mathbb {E} \left(\ln \left(\frac {\boldsymbol {\nu} _ {T , i}}{\boldsymbol {\nu} _ {0 , i}}\right) - T \ln \beta_ {2}\right). \tag {16} \\ \end{array}
$$
Then, according to Cauchy's inequality, we have
$$
\left(\mathbb {E} \sum_ {t = 1} ^ {T} \| \boldsymbol {G} _ {t} \| _ {1}\right) ^ {2} \leq \left(\sum_ {t = 1} ^ {T} \mathbb {E} \left[ \left\| \frac {1}{\sqrt [ 4 ]{\tilde {\nu} _ {t} ^ {1}}} \odot \boldsymbol {G} _ {t} \right\| ^ {2} \right]\right) \left(\sum_ {t = 1} ^ {T} \mathbb {E} \left[ \left\| \sqrt [ 4 ]{\tilde {\nu} _ {t} ^ {1}} \right\| ^ {2} \right]\right). \tag {17}
$$
Meanwhile, by Lemma 3, we have
$$
\sum_ {t = 1} ^ {T} \sum_ {i = 1} ^ {d} \mathbb {E} \sqrt {\tilde {\nu} _ {t , i}} \leq 2 (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\boldsymbol {\nu} _ {0 , i} + \sigma_ {0} ^ {2}} + 2 4 d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} \ln d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} + \frac {1 2 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} C _ {2}.
$$
Combining the above inequality and Eq. (17) gives
$$
\begin{array}{l} \left(\mathbb {E} \sum_ {t = 1} ^ {T} \| \boldsymbol {G} _ {t} \| _ {1}\right) ^ {2} \\ \leq \left(\frac {1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}}{1 - \beta_ {1}} \frac {8}{\eta} f (\boldsymbol {u} _ {1}) + \frac {3 2}{\beta_ {2} \left(1 - \frac {\beta_ {1}}{\sqrt {\beta_ {2}}}\right) ^ {2}} \sum_ {i = 1} ^ {d} \mathbb {E} \frac {\boldsymbol {G} _ {1 , i} ^ {2}}{\sqrt {\tilde {\nu} _ {1 , i}}} + C _ {1} \sum_ {i = 1} ^ {d} \left(\ln \left(\frac {\boldsymbol {\nu} _ {T , i}}{\boldsymbol {\nu} _ {0 , i}}\right) - T \ln \beta_ {2}\right)\right) \\ \times \left(2 (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\boldsymbol {\nu} _ {0 , i} + \sigma_ {0} ^ {2}} + 2 4 d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} \ln d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} + \frac {1 2 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} C _ {2}\right) \\ \leq \left(C _ {2} + 2 C _ {1} \sum_ {i = 1} ^ {d} \left(\ln \left(\sum_ {t = 1} ^ {T} \sum_ {i = 1} ^ {d} \mathbb {E} \sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}\right)\right)\right) \times \left(2 (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\boldsymbol {\nu} _ {0 , i} + \sigma_ {0} ^ {2}} + 2 4 d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} \ln d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} + \frac {1 2 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} C _ {2}\right) \\ \leq \left(C _ {2} + 2 C _ {1} \sum_ {i = 1} ^ {d} \left(\ln \left(2 (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\nu_ {0 , i} + \sigma_ {0} ^ {2}} + 2 4 d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} \ln d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} + \frac {1 2 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} C _ {2}\right)\right)\right) \\ \times \left(2 (T + 1) \sum_ {i = 1} ^ {d} \sqrt {\boldsymbol {\nu} _ {0 , i} + \sigma_ {0} ^ {2}} + 2 4 d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} \ln d \frac {\sigma_ {1} ^ {2} C _ {1}}{\sqrt {\beta_ {2}}} + \frac {1 2 \sigma_ {1} ^ {2}}{\sqrt {\beta_ {2}}} C _ {2}\right). \\ \end{array}
$$
The proof is then completed.
Proof. To start with, we have that
$$
\begin{array}{l} \frac {\left| \boldsymbol {G} _ {t , i} \right| ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \geq \frac {\frac {1}{2 \sigma_ {1} ^ {2}} \mathbb {E} ^ {\left| \mathcal {F} _ {t} \right|} \left| \boldsymbol {g} _ {t , i} \right| ^ {2}}{\sqrt {\tilde {\boldsymbol {\nu}} _ {t , i}}} \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \\ = \frac {\frac {1}{2 \sigma_ {1} ^ {2}} \mathbb {E} | \mathcal {F} _ {t} | \boldsymbol {g} _ {t , i} | ^ {2}}{\sqrt {\beta_ {2} \boldsymbol {\nu} _ {t - 1 , i} + (1 - \beta_ {2}) \sigma_ {0} ^ {2}}} \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}} \\ \geq \frac {1}{2 \sigma_ {1} ^ {2} \sqrt {1 - \beta_ {2}}} _ {\mathbb {E}} ^ {| \mathcal {F} _ {t}} \frac {| \boldsymbol {g} _ {t , i} | ^ {2}}{\sqrt {\frac {\nu_ {0 , i}}{1 - \beta_ {2}} + \sum_ {s = 1} ^ {T} | g _ {s , i} | ^ {2} + \sigma_ {0} ^ {2}}} \mathbf {1} _ {| G _ {t, i} | \geq \frac {\sigma_ {0}}{\sigma_ {1}}}, \\ \end{array}
$$
where the last inequality is due to that
$$
\beta_ {2} \boldsymbol {\nu} _ {t - 1, i} = (1 - \beta_ {2}) \sum_ {s = 1} ^ {t - 1} \beta_ {2} ^ {t - s} | \boldsymbol {g} _ {s, i} | ^ {2} + \beta_ {2} ^ {t} \boldsymbol {\nu} _ {0, i} \leq (1 - \beta_ {2}) \sum_ {s = 1} ^ {T} | \boldsymbol {g} _ {s, i} | ^ {2} + \boldsymbol {\nu} _ {0, i}. \tag {18}
$$
$$
\begin{array}{l} \frac {\sigma_ {0} ^ {2} + \frac {\boldsymbol {\nu} _ {0 , i}}{1 - \beta_ {2}}}{\sqrt {\frac {\boldsymbol {\nu} _ {0 , i}}{1 - \beta_ {2}} + \sum_ {s = 1} ^ {T} | g _ {s , i} | ^ {2} + \sigma_ {0} ^ {2}}} + \sum_ {t = 1} ^ {T} \mathbb {E} \frac {| \boldsymbol {g} _ {t , i} | ^ {2}}{\sqrt {\frac {\boldsymbol {\nu} _ {0 , i}}{1 - \beta_ {2}} + \sum_ {s = 1} ^ {T} | g _ {s , i} | ^ {2} + \sigma_ {0} ^ {2}}} \mathbf {1} _ {| \boldsymbol {G} _ {t, i} | <   \frac {\sigma_ {0}}{\sigma_ {1}}} \\ \leq \frac {\sigma_ {0} ^ {2} + \frac {\boldsymbol {\nu} _ {0 , i}}{1 - \beta_ {2}}}{\sqrt {\frac {\boldsymbol {\nu} _ {0 , i}}{1 - \beta_ {2}} + \sum_ {s = 1} ^ {T} | g _ {s , i} | ^ {2} \mathbf {1} _ {| \boldsymbol {G} _ {s , i} | <   \frac {\sigma_ {0}}{\sigma_ {1}} + \sigma_ {0} ^ {2}}} + \sum_ {t = 1} ^ {T} \mathbb {E}} \frac {| \boldsymbol {g} _ {t , i} | ^ {2}}{\sqrt {\frac {\boldsymbol {\nu} _ {0 , i}}{1 - \beta_ {2}} + \sum_ {s = 1} ^ {T} | g _ {s , i} | ^ {2} \mathbf {1} _ {| \boldsymbol {G} _ {s , i} | <   \frac {\sigma_ {0}}{\sigma_ {1}} + \sigma_ {0} ^ {2}}} \mathbf {1} _ {| \boldsymbol {G} _ {t, i} | <   \frac {\sigma_ {0}}{\sigma_ {1}}} \\ = \mathbb {E} \sqrt {\frac {\boldsymbol {\nu} _ {0 , i}}{1 - \beta_ {2}} + \sum_ {s = 1} ^ {T} | g _ {s , i} | ^ {2} \mathbf {1} _ {| \boldsymbol {G} _ {s , i} | <   \frac {\sigma_ {0}}{\sigma_ {1}}} + \sigma_ {0} ^ {2}} \leq \sqrt {\frac {\boldsymbol {\nu} _ {0 , i}}{1 - \beta_ {2}} + \mathbb {E} \sum_ {s = 1} ^ {T} | g _ {s , i} | ^ {2} \mathbf {1} _ {| \boldsymbol {G} _ {s , i} | <   \frac {\sigma_ {0}}{\sigma_ {1}}} + \sigma_ {0} ^ {2}} \\ \leq \sqrt {\frac {\nu_ {0 , i}}{1 - \beta_ {2}} + 2 \sigma_ {0} ^ {2} T + \sigma_ {0} ^ {2}}. \tag {19} \\ \end{array}
$$
Conclusively, we obtain