SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} \left\| \nabla F _ {t, \eta} (x _ {t}, y _ {t} ^ {*} (x _ {t})) - \widehat {\nabla} F _ {t, \eta} (x _ {t}, y _ {t + 1}) \right\| ^ {2} \\ \leq \frac {3 L _ {1} ^ {2}}{W} \sum_ {t = 2} ^ {T} \sum_ {i = 0} ^ {K - 1} \eta^ {i} \left\{\left(1 - \gamma \mu_ {g}\right) ^ {t - 1 - i} \delta_ {1} \right. \\ + G _ {2} \sum_ {j = 0} ^ {t - i - 2} (1 - \gamma \mu_ {g}) ^ {j} \| y _ {t - 1 - i - j} ^ {*} (x _ {t - 1 - i - j}) - y _ {t - i - j} ^ {*} (x _ {t - 1 - i - j}) \| ^ {2} \\ \left. + \frac {2 L _ {1} ^ {2} G _ {2}}{\mu_ {g} ^ {2}} \sum_ {j = 0} ^ {t - 2 - i} (1 - \gamma \mu_ {g}) ^ {j} \| x _ {t - 1 - i - j} - x _ {t - i - j} \| ^ {2} \right\} + \| \nabla f _ {1} \left(x _ {1}, y _ {1} ^ {*} \left(x _ {1}\right)\right) - \widehat {\nabla} f _ {1} \left(x _ {1}, y _ {2}\right) \| ^ {2} \\ \leq \frac {3 L _ {1} ^ {2}}{W} \left[ \frac {\delta_ {1} \eta (1 - \gamma \mu_ {g})}{(1 - \eta) (\eta - 1 + \gamma \mu_ {g})} + G _ {2} \frac {\eta}{(1 - \eta) (\eta - 1 + \gamma \mu_ {g})} H _ {2, T} \right. \\ + \frac {2 L _ {1} ^ {2} G _ {2}}{\mu_ {g} ^ {2}} \frac {2 \beta^ {2} \eta}{(1 - \eta) (\eta - 1 + \gamma \mu_ {g})} \sum_ {t = 1} ^ {T} \| \nabla F _ {t} (x _ {t}, y _ {t} ^ {*} (x _ {t})) - \widehat {\nabla} F _ {t} (x _ {t}, y _ {t + 1}) \| ^ {2} \\ \left. + \frac {2 L _ {1} ^ {2} G _ {2}}{\mu_ {g} ^ {2}} \frac {2 \beta^ {2} \eta}{(1 - \eta) (\eta - 1 + \gamma \mu_ {g})} \sum_ {t = 1} ^ {T} \| \nabla F _ {t} (x _ {t}, y _ {t} ^ {*} (x _ {t})) \| ^ {2} \right] + \| \nabla f _ {1} (x _ {1}, y _ {1} ^ {*} (x _ {1})) - \widehat {\nabla} f _ {1} (x _ {1}, y _ {2}) \| ^ {2} \\ \end{array}
$$
which gives that
$$
\begin{array}{l} \left(1 - \frac {1 2 L _ {1} ^ {4} \beta^ {2} G _ {2} \eta}{\mu_ {g} ^ {2} W (1 - \eta) (\eta - 1 + \gamma \mu_ {g})}\right) \sum_ {t = 1} ^ {T} \| \nabla F _ {t, \eta} (x _ {t}, y _ {t} ^ {*} (x _ {t})) - \widehat {\nabla} F _ {t, \eta} (x _ {t}, y _ {t + 1}) \| ^ {2} \\ \leq \frac {3 L _ {1} ^ {2} \eta}{W (1 - \eta) (\eta - 1 + \gamma \mu_ {g})} \left[ \delta_ {1} (1 - \gamma \mu_ {g}) + G _ {2} H _ {2, T} + \frac {4 L _ {1} ^ {2} G _ {2} \beta^ {2}}{\mu_ {g} ^ {2}} \sum_ {t = 1} ^ {T} \| \nabla F _ {t} (x _ {t}, y _ {t} ^ {*} (x _ {t})) \| ^ {2} \right] \\ + \left\| \nabla f _ {1} \left(x _ {1}, y _ {1} ^ {*} \left(x _ {1}\right)\right) - \widehat {\nabla} f _ {1} \left(x _ {1}, y _ {2}\right) \right\| ^ {2}. \\ \end{array}
$$
Here  $1 - \frac{12L_1^4\beta^2G_2\eta}{\mu_g^2W(1 - \eta)(\eta - 1 + \gamma\mu_g)}\geq \frac{1}{2}$  because
$$
\begin{array}{l} \beta^ {2} \leq \frac {\beta}{L _ {f}} \leq \frac {\mu_ {g} ^ {2} L _ {f} W (1 - \eta) (\eta - 1 - \alpha \mu_ {g} / 2)}{2 4 L _ {1} ^ {4} G _ {2} \eta} L _ {f} \\ = \frac {\mu_ {g} ^ {2} W (1 - \eta) (\eta - 1 - \alpha \mu_ {g} / 2)}{2 4 L _ {1} ^ {4} G _ {2} \eta}. \\ \end{array}
$$
Let  $G_{3} = 1 - \frac{12L_{1}^{4}\beta^{2}G_{2}\eta}{\mu_{g}^{2}W(1 - \eta)(\eta - 1 + \gamma\mu_{g})}$  and  $G_{4} = \frac{3L_{1}^{2}\eta}{W(1 - \eta)(\eta - 1 + \gamma\mu_{g})}$ . It is clear that
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} \left\| \nabla F _ {t, \eta} \left(x _ {t}, y _ {t} ^ {*} \left(x _ {t}\right)\right) - \widehat {\nabla} F _ {t, \eta} \left(x _ {t}, y _ {t + 1}\right) \right\| ^ {2} \\ \leq \frac {G _ {4}}{G _ {3}} \left[ \delta_ {1} (1 - \gamma \mu_ {g}) + G _ {2} H _ {2, T} \right] + \frac {G _ {4}}{G _ {3}} \frac {4 L _ {1} ^ {2} G _ {2} \beta^ {2}}{\mu_ {g} ^ {2}} \sum_ {t = 1} ^ {T} \| \nabla F _ {t} (x _ {t}, y _ {t} ^ {*} (x _ {t})) \| ^ {2} \\ \end{array}
$$
$$
+ \frac {1}{G _ {3}} \| \nabla f _ {1} (x _ {1}, y _ {1} ^ {*} (x _ {1})) - \widehat {\nabla} f _ {1} (x _ {1}, y _ {2}) \| ^ {2}.
$$
Based on Equation (11), we can have
$$
\begin{array}{l} \left(\frac {\beta}{2} - \beta^ {2} L _ {f}\right) \sum_ {t = 1} ^ {T} \| \nabla F _ {t, \eta} (x _ {t}, y _ {t} ^ {*} (x _ {t})) \| ^ {2} \\ \leq \sum_ {t = 1} ^ {T} \left[ F _ {t, \eta} \left(x _ {t}, y _ {t} ^ {*} \left(x _ {t}\right)\right) - F _ {t, \eta} \left(x _ {t + 1}, y _ {t} ^ {*} \left(x _ {t + 1}\right)\right) \right] \\ + \left(\frac {\beta}{2} + \beta^ {2} L _ {f}\right) \sum_ {t = 1} ^ {T} \| \nabla F _ {t, \eta} \left(x _ {t}, y _ {t} ^ {*} \left(x _ {t}\right)\right) - \widehat {\nabla} F _ {t, \eta} \left(x _ {t}, y _ {t + 1}\right) \| ^ {2} \\ \leq \sum_ {t = 1} ^ {T} \left[ F _ {t, \eta} \left(x _ {t}, y _ {t} ^ {*} \left(x _ {t}\right)\right) - F _ {t, \eta} \left(x _ {t + 1}, y _ {t} ^ {*} \left(x _ {t + 1}\right)\right) \right] + \left(\frac {\beta}{2} + \beta^ {2} L _ {f}\right) \left[ \frac {G _ {4}}{G _ {3}} \left[ \delta_ {1} \left(1 - \gamma \mu_ {g}\right) + G _ {2} H _ {2, T} \right] \right. \\ \left. + \frac {G _ {4}}{G _ {3}} \frac {4 L _ {1} ^ {2} G _ {2} \beta^ {2}}{\mu_ {g} ^ {2}} \sum_ {t = 1} ^ {T} \| \nabla F _ {t} \left(x _ {t}, y _ {t} ^ {*} \left(x _ {t}\right)\right) \| ^ {2} + \frac {1}{G _ {3}} \| \nabla f _ {1} \left(x _ {1}, y _ {1} ^ {*} \left(x _ {1}\right)\right) - \widehat {\nabla} f _ {1} \left(x _ {1}, y _ {2}\right) \| ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} \| \nabla F _ {t, \eta} (x _ {t}, y _ {t} ^ {*} (x _ {t})) \| ^ {2} \\ \leq \frac {1}{\frac {\beta}{2} - \beta^ {2} L _ {f} - \frac {G _ {4}}{G _ {3}} \frac {4 L _ {1} ^ {2} G _ {2} \beta^ {2}}{\mu_ {g} ^ {2}} \left(\frac {\beta}{2} + \beta^ {2} L _ {f}\right)} \sum_ {t = 1} ^ {T} [ F _ {t, \eta} (x _ {t}, y _ {t} ^ {*} (x _ {t})) - F _ {t, \eta} (x _ {t + 1}, y _ {t} ^ {*} (x _ {t + 1})) ] \\ + \frac {\left(\frac {1}{2} + \beta L _ {f}\right) \left\{\frac {G _ {4}}{G _ {3}} [ \delta_ {1} (1 - \gamma \mu_ {g}) + G _ {2} H _ {2 , T} ] + \frac {1}{G _ {3}} \| \nabla f _ {1} (x _ {1} , y _ {1} ^ {*} (x _ {1})) - \widehat {\nabla} f _ {1} (x _ {1} , y _ {2}) \| ^ {2} \right\}}{\frac {1}{2} - \beta L _ {f} - \frac {G _ {4}}{G _ {3}} \frac {4 L _ {1} ^ {2} G _ {2} \beta}{\mu_ {g} ^ {2}} \left(\frac {\beta}{2} + \beta^ {2} L _ {f}\right)} \\ \leq \frac {\frac {2 M T}{W} + V _ {1 , T}}{\frac {\beta}{2} - \beta^ {2} L _ {f} - \frac {G _ {4}}{G _ {3}} \frac {4 L _ {1} ^ {2} G _ {2} \beta^ {2}}{\mu_ {g} ^ {2}} \left(\frac {\beta}{2} + \beta^ {2} L _ {f}\right)} + \frac {\left(\frac {1}{2} + \beta L _ {f}\right) \frac {G _ {4} G _ {2} H _ {2 , T}}{G _ {3}}}{\frac {1}{2} - \beta L _ {f} - \frac {G _ {4}}{G _ {3}} \frac {4 L _ {1} ^ {2} G _ {2} \beta}{\mu_ {g} ^ {2}} \left(\frac {\beta}{2} + \beta^ {2} L _ {f}\right)} \\ + \frac {\left(\frac {1}{2} + \beta L _ {f}\right) \left\{\frac {G _ {4}}{G _ {3}} \delta_ {1} (1 - \gamma \mu_ {g}) + \frac {1}{G _ {3}} \| \nabla f _ {1} (x _ {1} , y _ {1} ^ {*} (x _ {1})) - \widehat {\nabla} f _ {1} (x _ {1} , y _ {2}) \| ^ {2} \right\}}{\frac {1}{2} - \beta L _ {f} - \frac {G _ {4}}{G _ {3}} \frac {4 L _ {1} ^ {2} G _ {2} \beta}{\mu_ {g} ^ {2}} \left(\frac {\beta}{2} + \beta^ {2} L _ {f}\right)} \\ = O \left(\frac {T}{\beta W} + \frac {V _ {1 , T}}{\beta} + H _ {2, T}\right). \\ \end{array}
$$
Here  $\frac{1}{2} -\beta L_{f} - \frac{G_{4}}{G_{3}}\frac{4L_{1}^{2}G_{2}\beta}{\mu_{g}^{2}}\left(\frac{\beta}{2} +\beta^{2}L_{f}\right) > 0$  because
$$
\begin{array}{l} \frac {1}{2} - \beta L _ {f} - \frac {G _ {4}}{G _ {3}} \frac {4 L _ {1} ^ {2} G _ {2} \beta}{\mu_ {g} ^ {2}} \left(\frac {\beta}{2} + \beta^ {2} L _ {f}\right) \\ \stackrel {(a)} {>} \frac {1}{2} - \beta L _ {f} - \frac {G _ {4}}{G _ {3}} \frac {4 L _ {1} ^ {2} G _ {2} \beta^ {2}}{\mu_ {g} ^ {2}} \\ \stackrel {(b)} {\geq} \frac {1}{2} - 2 \beta L _ {f} \\ \begin{array}{l} \stackrel {(c)} {\geq 0}, \end{array} \\ \end{array}
$$
where (a) and (c) are because  $\beta L_f < \frac{1}{4}$ , and (b) is because  $\beta L_f > \frac{G_4}{G_3} \frac{4L_1^2 G_2 \beta^2}{\mu_g^2}$ .